fix bugs in fetch and add support for no limit on v2 list

Author: tonycai653

cmd/root.go

@@ -86,5 +86,9 @@ func initConfig() {
 		viper.AddConfigPath(curUser.HomeDir)
 		viper.SetConfigName(".qshell")
 	}
-	viper.ReadInConfig()
+	if rErr := viper.ReadInConfig(); rErr != nil {
+		if _, ok := rErr.(viper.ConfigFileNotFoundError); !ok {
+			fmt.Fprintf(os.Stderr, "read config file: %v\n", rErr)
+		}
+	}
 }

cmd/rs.go

@@ -144,7 +144,7 @@ func init() {
 	lsBucketCmd2.Flags().StringVarP(&listMarker, "marker", "m", "", "list marker")
 	lsBucketCmd2.Flags().StringVarP(&prefix, "prefix", "p", "", "list by prefix")
 	lsBucketCmd2.Flags().StringVarP(&suffixes, "suffixes", "q", "", "list by key suffixes, separated by comma")
-	lsBucketCmd2.Flags().IntVarP(&maxRetry, "max-retry", "x", 5, "max retries when error occurred")
+	lsBucketCmd2.Flags().IntVarP(&maxRetry, "max-retry", "x", 20, "max retries when error occurred")
 	lsBucketCmd2.Flags().StringVarP(&outFile, "out", "o", "", "output file")
 	lsBucketCmd2.Flags().StringVarP(&startDate, "start", "s", "", "start date with format yyyy-mm-dd-hh-MM-ss")
 	lsBucketCmd2.Flags().StringVarP(&endDate, "end", "e", "", "end date with format yyyy-mm-dd-hh-MM-ss")

cmd/rsbatch.go

@@ -109,7 +109,10 @@ var (
 
 func init() {
 	batchFetchCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "urls list file")
-	batchFetchCmd.Flags().IntVarP(&worker, "worker", "c", 5, "worker count")
+	batchFetchCmd.Flags().IntVarP(&worker, "worker", "c", 1, "worker count")
+	batchFetchCmd.Flags().StringVarP(&bsuccessFname, "success-list", "s", "", "file to save batch fetch success list")
+	batchFetchCmd.Flags().StringVarP(&bfailureFname, "failure-list", "e", "", "file to save batch fetch failure list")
+
 	batchStatCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "input file")
 	batchCopyCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "input file")
 	batchMoveCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "input file")
@@ -195,11 +198,16 @@ func BatchFetch(cmd *cobra.Command, params []string) {
 	)
 	defer close(fItemChan)
 
-	go batchFetch(fItemChan)
+	fileExporter, fErr := iqshell.NewFileExporter(bsuccessFname, bfailureFname, "")
+	if fErr != nil {
+		fmt.Fprintf(os.Stderr, "create file exporter: %v\n", fErr)
+		os.Exit(1)
+	}
+	go batchFetch(fItemChan, fileExporter)
 
 	for scanner.Scan() {
 		line := scanner.Text()
-		items := strings.Fields(line)
+		items := strings.Split(line, "\t")
 		if len(items) <= 0 {
 			continue
 		}
@@ -225,13 +233,24 @@ func BatchFetch(cmd *cobra.Command, params []string) {
 	}
 }
 
-func batchFetch(fItemChan chan *iqshell.FetchItem) {
+func batchFetch(fItemChan chan *iqshell.FetchItem, fileExporter *iqshell.FileExporter) {
 	for i := 0; i < worker; i++ {
 		go func() {
 			bm := iqshell.GetBucketManager()
 			for fetchItem := range fItemChan {
-				fetchResult, fErr := bm.Fetch(fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
-				fmt.Println(fetchResult, fErr)
+				_, fErr := bm.Fetch(fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
+				if fErr != nil {
+					fmt.Fprintf(os.Stderr, "fetch %s => %s:%s failed\n", fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
+					if fileExporter != nil {
+						fileExporter.WriteToFailedWriter(fmt.Sprintf("%s\t%s\t%v\n", fetchItem.RemoteUrl, fetchItem.Key, fErr))
+					}
+				} else {
+					fmt.Printf("fetch %s => %s:%s success\n", fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
+					if fileExporter != nil {
+						fileExporter.WriteToSuccessWriter(fmt.Sprintf("%s\t%s\n", fetchItem.RemoteUrl, fetchItem.Key))
+					}
+
+				}
 			}
 		}()
 	}

docs/batchcopy.md

@@ -14,7 +14,7 @@ qshell batchcopy [--force] [--overwrite] [--success-list <SuccessFileName>] [--f
 # 帮助
 ```
 qshell batchcopy -h
-``**
+```
 
 # 鉴权
 

docs/batchdelete.md

@@ -11,7 +11,7 @@ qshell batchdelete [--force] [--success-list <SuccessFileName>] [--failure-list
 # 帮助
 ```
 qshell batchdelete -h
-``**
+```
 
 # 鉴权
 

docs/batchexpire.md

@@ -12,7 +12,7 @@ qshell batchexpire [--force] <Bucket> <-i KeyDeleteAfterDaysMapFile>
 # 帮助 
 ```
 qshell batchexpire -h
-``**
+```
 
 # 鉴权
 

docs/batchrename.md

@@ -9,7 +9,7 @@
 
 ```
 qshell batchrename [--force] [--overwrite] [--success-list <SuccessFileName>] [--failure-list <failureFileName>] <Bucket> [-i <OldNewKeyMapFile>]
-`****
+```
 
 # 鉴权
 

docs/batchsign.md

@@ -6,7 +6,7 @@
 
 ```
 qshell batchsign [<-i UrlListFile>] [-e <Deadline>]
-``**
+```
 
 # 鉴权
 

docs/batchstat.md

@@ -6,7 +6,7 @@
 
 ```
 qshell batchstat <Bucket> <-i KeyListFile>
-``**
+```
 
 # 鉴权
 

docs/listbucket2.md

@@ -15,9 +15,11 @@ Key\tSize\tHash\tPutTime\tMimeType\tFileType\tEndUser
 # 格式
 
 ```
-qshell listbucket2 [--prefix <Prefix> | --suffixes <suffixes1,suffixes2>] [--start <StartDate>] [--end <EndDate>] <Bucket> [-o <ListBucketResultFile>]
+qshell listbucket2 [--prefix <Prefix> | --suffixes <suffixes1,suffixes2>] [--start <StartDate>] [--max-retry <RetryCount>][--end <EndDate>] <Bucket> [-o <ListBucketResultFile>]
 ```
 
+选项max-retry， 默认列举出错的重试次数为20次，如果希望可以列举完文件列表，不限错误次数，可以设置max-retry为负数。
+
 （1）获取空间中所有的文件列表，这种情况下，可以直接指定 `Bucket` 参数和结果保存文件参数 `ListBucketResultFile` 即可。
 
 ```