copyurl: Added --url to read urls from csv file - #8127

cmd/copyurl/copyurl.go

@@ -3,6 +3,7 @@ package copyurl
 
 import (
 	"context"
+	"encoding/csv"
 	"errors"
 	"fmt"
 	"os"
@@ -21,6 +22,7 @@ var (
 	printFilename  = false
 	stdout         = false
 	noClobber      = false
+	urls           = false
 )
 
 func init() {
@@ -31,6 +33,7 @@ func init() {
 	flags.BoolVarP(cmdFlags, &printFilename, "print-filename", "p", printFilename, "Print the resulting name from --auto-filename", "")
 	flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name", "")
 	flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file", "")
+	flags.BoolVarP(cmdFlags, &urls, "urls", "", stdout, "Use a .csv file of links to process multiple urls", "")
 }
 
 var commandDefinition = &cobra.Command{
@@ -54,6 +57,10 @@ destination if there is one with the same name.
 Setting |--stdout| or making the output file name |-|
 will cause the output to be written to standard output.
 
+Setting |--urls| allows you to input a .csv file of urls in format:
+URL,FILENAME. If a filename is missing one will be autogenerated for the file,
+and it will otherwise continue as normal.
+
 ### Troubleshooting
 
 If you can't get |rclone copyurl| to work then here are some things you can try:
@@ -68,33 +75,101 @@ If you can't get |rclone copyurl| to work then here are some things you can try:
 		"groups":            "Important",
 	},
 	RunE: func(command *cobra.Command, args []string) (err error) {
+
+		//Check they gave all the args correctly
 		cmd.CheckArgs(1, 2, command, args)
 
-		var dstFileName string
-		var fsdst fs.Fs
+		//Create list to store the list of URLS
+		var urlList [][]string
+
+		if urls {
+			//Read the .csv file provided if --urls is enabled
+			File, error := os.Open(args[0])
+			if error != nil {
+				return errors.New("failed to open .csv file")
+			}
+			reader := csv.NewReader(File)
+			reader.FieldsPerRecord = -1
+			data, error := reader.ReadAll()
+			if error != nil {
+				return errors.New("failed reading .csv file")
+			}
+			//Save the list of urls to the urlList variable
+			urlList = data
+
+		} else {
+			//If its the non multi-url command, save only that, without attempting to read a file
+			urlList = make([][]string, 1)
+			urlList[0] = args
+		}
+
+		//Save per-file variables
+		dstFileNames := make([]string, len(urlList))
+		fsdsts := make([]fs.Fs, len(urlList))
+		autonames := make([]bool, len(urlList))
+
+		//Save the original input folder, prevents it being overwritten when adding onto the end
+		root := args[1]
+
 		if !stdout {
 			if len(args) < 2 {
 				return errors.New("need 2 arguments if not using --stdout")
 			}
 			if args[1] == "-" {
 				stdout = true
-			} else if autoFilename {
-				fsdst = cmd.NewFsDir(args[1:])
 			} else {
-				fsdst, dstFileName = cmd.NewFsDstFile(args[1:])
+
+				//Go over each of the URLs need transferring
+				for i := range urlList {
+					//Save autoFilenames
+					if autoFilename {
+						autonames[i] = true
+					} else if urls {
+
+						//If we are using multiple URLs, and they dont provide a name for one, set it to autogenerate one for it.
+						//This is because itd suck having it cancel midway through if you forgot to add the name of one.
+
+						autonames[i] = (len(urlList[i]) == 1 || urlList[i][1] == "")
+					}
+
+					if autonames[i] {
+						//If it is autonaming, generate the name.
+						//Set args[1] to the root, resetting it to the main directory. (Clears filenames from it)
+						args[1] = root
+						fsdsts[i] = cmd.NewFsDir(args[1:])
+					} else {
+						//If using multiple URLs, set it to the root and the name.
+						//Because again, args[1] gets appended to each round, so it has to be reset.
+						if len(urlList) > 1 {
+							args[1] = root + urlList[i][1]
+						}
+						//Otherwise your clear to go, as in the regular format,
+						//args[1] already contains the destination and filename for a single file
+						fsdsts[i], dstFileNames[i] = cmd.NewFsDstFile(args[1:])
+					}
+				}
 			}
 		}
 		cmd.Run(true, true, command, func() error {
-			var dst fs.Object
-			if stdout {
-				err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout)
-			} else {
-				dst, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, headerFilename, noClobber)
-				if printFilename && err == nil && dst != nil {
-					fmt.Println(dst.Remote())
+			//Go over each URL, to upload them
+			for i, url := range urlList {
+				//Genereate a new fs.Object for the file
+				var dst fs.Object
+				if stdout {
+					err = operations.CopyURLToWriter(context.Background(), url[0], os.Stdout)
+				} else {
+					//CopyURLs, with each individual link's information
+					dst, err = operations.CopyURL(context.Background(), fsdsts[i], dstFileNames[i], url[0], autonames[i], headerFilename, noClobber)
+					if printFilename && err == nil && dst != nil {
+						fmt.Println(dst.Remote())
+					}
+				}
+				if err != nil {
+					println(err)
+					return err
 				}
 			}
-			return err
+			return nil
 		})
 		return nil
 	},