diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index aae34e496..c15f4bed4 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -3,6 +3,7 @@ package copyurl import ( "context" + "encoding/csv" "errors" "fmt" "os" @@ -21,6 +22,7 @@ var ( printFilename = false stdout = false noClobber = false + urls = false ) func init() { @@ -31,6 +33,7 @@ func init() { flags.BoolVarP(cmdFlags, &printFilename, "print-filename", "p", printFilename, "Print the resulting name from --auto-filename", "") flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name", "") flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file", "") + flags.BoolVarP(cmdFlags, &urls, "urls", "", stdout, "Use a .csv file of links to process multiple urls", "") } var commandDefinition = &cobra.Command{ @@ -54,6 +57,10 @@ destination if there is one with the same name. Setting |--stdout| or making the output file name |-| will cause the output to be written to standard output. +Setting |--urls| allows you to input a .csv file of urls in format: +URL,FILENAME. If a filename is missing one will be autogenerated for the file, +and it will otherwise continue as normal. + ### Troubleshooting If you can't get |rclone copyurl| to work then here are some things you can try: @@ -68,33 +75,101 @@ If you can't get |rclone copyurl| to work then here are some things you can try: "groups": "Important", }, RunE: func(command *cobra.Command, args []string) (err error) { + + //Check they gave all the args correctly cmd.CheckArgs(1, 2, command, args) - var dstFileName string - var fsdst fs.Fs + //Create list to store the list of URLS + var urlList [][]string + + if urls { + //Read the .csv file provided if --urls is enabled + File, error := os.Open(args[0]) + if error != nil { + return errors.New("failed to open .csv file") + } + reader := csv.NewReader(File) + reader.FieldsPerRecord = -1 + data, error := reader.ReadAll() + if error != nil { + return errors.New("failed reading .csv file") + } + //Save the list of urls to the urlList variable + urlList = data + + } else { + //If its the non multi-url command, save only that, without attempting to read a file + urlList = make([][]string, 1) + urlList[0] = args + } + + //Save per-file variables + dstFileNames := make([]string, len(urlList)) + fsdsts := make([]fs.Fs, len(urlList)) + autonames := make([]bool, len(urlList)) + + //Save the original input folder, prevents it being overwritten when adding onto the end + root := args[1] + if !stdout { if len(args) < 2 { return errors.New("need 2 arguments if not using --stdout") } if args[1] == "-" { stdout = true - } else if autoFilename { - fsdst = cmd.NewFsDir(args[1:]) } else { - fsdst, dstFileName = cmd.NewFsDstFile(args[1:]) + + //Go over each of the URLs need transferring + for i := range urlList { + //Save autoFilenames + if autoFilename { + autonames[i] = true + } else if urls { + + //If we are using multiple URLs, and they dont provide a name for one, set it to autogenerate one for it. + //This is because itd suck having it cancel midway through if you forgot to add the name of one. + + autonames[i] = (len(urlList[i]) == 1 || urlList[i][1] == "") + } + + if autonames[i] { + //If it is autonaming, generate the name. + //Set args[1] to the root, resetting it to the main directory. (Clears filenames from it) + args[1] = root + fsdsts[i] = cmd.NewFsDir(args[1:]) + } else { + //If using multiple URLs, set it to the root and the name. + //Because again, args[1] gets appended to each round, so it has to be reset. + if len(urlList) > 1 { + args[1] = root + urlList[i][1] + } + //Otherwise your clear to go, as in the regular format, + //args[1] already contains the destination and filename for a single file + fsdsts[i], dstFileNames[i] = cmd.NewFsDstFile(args[1:]) + } + } } } cmd.Run(true, true, command, func() error { - var dst fs.Object - if stdout { - err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout) - } else { - dst, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, headerFilename, noClobber) - if printFilename && err == nil && dst != nil { - fmt.Println(dst.Remote()) + //Go over each URL, to upload them + for i, url := range urlList { + //Genereate a new fs.Object for the file + var dst fs.Object + if stdout { + err = operations.CopyURLToWriter(context.Background(), url[0], os.Stdout) + } else { + //CopyURLs, with each individual link's information + dst, err = operations.CopyURL(context.Background(), fsdsts[i], dstFileNames[i], url[0], autonames[i], headerFilename, noClobber) + if printFilename && err == nil && dst != nil { + fmt.Println(dst.Remote()) + } + } + if err != nil { + println(err) + return err } } - return err + return nil }) return nil },