test: move test commands under "rclone test" and make them visible
This commit is contained in:
Nick Craig-Wood
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
exec rclone --check-normalization=true --check-control=true --check-length=true info \
|
||||
/tmp/testInfo \
|
||||
TestAmazonCloudDrive:testInfo \
|
||||
TestB2:testInfo \
|
||||
TestCryptDrive:testInfo \
|
||||
TestCryptSwift:testInfo \
|
||||
TestDrive:testInfo \
|
||||
TestDropbox:testInfo \
|
||||
TestGoogleCloudStorage:rclone-testinfo \
|
||||
TestOneDrive:testInfo \
|
||||
TestS3:rclone-testinfo \
|
||||
TestSftp:testInfo \
|
||||
TestSwift:testInfo \
|
||||
TestYandex:testInfo \
|
||||
TestFTP:testInfo
|
||||
|
||||
# TestHubic:testInfo \
|
||||
@@ -0,0 +1,462 @@
|
||||
package info
|
||||
|
||||
// FIXME once translations are implemented will need a no-escape
|
||||
// option for Put so we can make these tests work again
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/test"
|
||||
"github.com/rclone/rclone/cmd/test/info/internal"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
writeJSON string
|
||||
checkNormalization bool
|
||||
checkControl bool
|
||||
checkLength bool
|
||||
checkStreaming bool
|
||||
all bool
|
||||
uploadWait time.Duration
|
||||
positionLeftRe = regexp.MustCompile(`(?s)^(.*)-position-left-([[:xdigit:]]+)$`)
|
||||
positionMiddleRe = regexp.MustCompile(`(?s)^position-middle-([[:xdigit:]]+)-(.*)-$`)
|
||||
positionRightRe = regexp.MustCompile(`(?s)^position-right-([[:xdigit:]]+)-(.*)$`)
|
||||
)
|
||||
|
||||
func init() {
|
||||
test.Command.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.StringVarP(cmdFlags, &writeJSON, "write-json", "", "", "Write results to file.")
|
||||
flags.BoolVarP(cmdFlags, &checkNormalization, "check-normalization", "", false, "Check UTF-8 Normalization.")
|
||||
flags.BoolVarP(cmdFlags, &checkControl, "check-control", "", false, "Check control characters.")
|
||||
flags.DurationVarP(cmdFlags, &uploadWait, "upload-wait", "", 0, "Wait after writing a file.")
|
||||
flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", false, "Check max filename length.")
|
||||
flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", false, "Check uploads with indeterminate file size.")
|
||||
flags.BoolVarP(cmdFlags, &all, "all", "", false, "Run all tests.")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "info [remote:path]+",
|
||||
Short: `Discovers file name or other limitations for paths.`,
|
||||
Long: `rclone info discovers what filenames and upload methods are possible
|
||||
to write to the paths passed in and how long they can be. It can take some
|
||||
time. It will write test files into the remote:path passed in. It outputs
|
||||
a bit of go code for each one.
|
||||
|
||||
**NB** this can create undeletable files and other hazards - use with care
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1e6, command, args)
|
||||
if !checkNormalization && !checkControl && !checkLength && !checkStreaming && !all {
|
||||
log.Fatalf("no tests selected - select a test or use -all")
|
||||
}
|
||||
if all {
|
||||
checkNormalization = true
|
||||
checkControl = true
|
||||
checkLength = true
|
||||
checkStreaming = true
|
||||
}
|
||||
for i := range args {
|
||||
f := cmd.NewFsDir(args[i : i+1])
|
||||
cmd.Run(false, false, command, func() error {
|
||||
return readInfo(context.Background(), f)
|
||||
})
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
type results struct {
|
||||
ctx context.Context
|
||||
f fs.Fs
|
||||
mu sync.Mutex
|
||||
stringNeedsEscaping map[string]internal.Position
|
||||
controlResults map[string]internal.ControlResult
|
||||
maxFileLength int
|
||||
canWriteUnnormalized bool
|
||||
canReadUnnormalized bool
|
||||
canReadRenormalized bool
|
||||
canStream bool
|
||||
}
|
||||
|
||||
func newResults(ctx context.Context, f fs.Fs) *results {
|
||||
return &results{
|
||||
ctx: ctx,
|
||||
f: f,
|
||||
stringNeedsEscaping: make(map[string]internal.Position),
|
||||
controlResults: make(map[string]internal.ControlResult),
|
||||
}
|
||||
}
|
||||
|
||||
// Print the results to stdout
|
||||
func (r *results) Print() {
|
||||
fmt.Printf("// %s\n", r.f.Name())
|
||||
if checkControl {
|
||||
escape := []string{}
|
||||
for c, needsEscape := range r.stringNeedsEscaping {
|
||||
if needsEscape != internal.PositionNone {
|
||||
k := strconv.Quote(c)
|
||||
k = k[1 : len(k)-1]
|
||||
escape = append(escape, fmt.Sprintf("'%s'", k))
|
||||
}
|
||||
}
|
||||
sort.Strings(escape)
|
||||
fmt.Printf("stringNeedsEscaping = []rune{\n")
|
||||
fmt.Printf("\t%s\n", strings.Join(escape, ", "))
|
||||
fmt.Printf("}\n")
|
||||
}
|
||||
if checkLength {
|
||||
fmt.Printf("maxFileLength = %d\n", r.maxFileLength)
|
||||
}
|
||||
if checkNormalization {
|
||||
fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized)
|
||||
fmt.Printf("canReadUnnormalized = %v\n", r.canReadUnnormalized)
|
||||
fmt.Printf("canReadRenormalized = %v\n", r.canReadRenormalized)
|
||||
}
|
||||
if checkStreaming {
|
||||
fmt.Printf("canStream = %v\n", r.canStream)
|
||||
}
|
||||
}
|
||||
|
||||
// WriteJSON writes the results to a JSON file when requested
|
||||
func (r *results) WriteJSON() {
|
||||
if writeJSON == "" {
|
||||
return
|
||||
}
|
||||
|
||||
report := internal.InfoReport{
|
||||
Remote: r.f.Name(),
|
||||
}
|
||||
if checkControl {
|
||||
report.ControlCharacters = &r.controlResults
|
||||
}
|
||||
if checkLength {
|
||||
report.MaxFileLength = &r.maxFileLength
|
||||
}
|
||||
if checkNormalization {
|
||||
report.CanWriteUnnormalized = &r.canWriteUnnormalized
|
||||
report.CanReadUnnormalized = &r.canReadUnnormalized
|
||||
report.CanReadRenormalized = &r.canReadRenormalized
|
||||
}
|
||||
if checkStreaming {
|
||||
report.CanStream = &r.canStream
|
||||
}
|
||||
|
||||
if f, err := os.Create(writeJSON); err != nil {
|
||||
fs.Errorf(r.f, "Creating JSON file failed: %s", err)
|
||||
} else {
|
||||
defer fs.CheckClose(f, &err)
|
||||
enc := json.NewEncoder(f)
|
||||
enc.SetIndent("", " ")
|
||||
err := enc.Encode(report)
|
||||
if err != nil {
|
||||
fs.Errorf(r.f, "Writing JSON file failed: %s", err)
|
||||
}
|
||||
}
|
||||
fs.Infof(r.f, "Wrote JSON file: %s", writeJSON)
|
||||
}
|
||||
|
||||
// writeFile writes a file with some random contents
|
||||
func (r *results) writeFile(path string) (fs.Object, error) {
|
||||
contents := random.String(50)
|
||||
src := object.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
|
||||
obj, err := r.f.Put(r.ctx, bytes.NewBufferString(contents), src)
|
||||
if uploadWait > 0 {
|
||||
time.Sleep(uploadWait)
|
||||
}
|
||||
return obj, err
|
||||
}
|
||||
|
||||
// check whether normalization is enforced and check whether it is
|
||||
// done on the files anyway
|
||||
func (r *results) checkUTF8Normalization() {
|
||||
unnormalized := "Héroique"
|
||||
normalized := "Héroique"
|
||||
_, err := r.writeFile(unnormalized)
|
||||
if err != nil {
|
||||
r.canWriteUnnormalized = false
|
||||
return
|
||||
}
|
||||
r.canWriteUnnormalized = true
|
||||
_, err = r.f.NewObject(r.ctx, unnormalized)
|
||||
if err == nil {
|
||||
r.canReadUnnormalized = true
|
||||
}
|
||||
_, err = r.f.NewObject(r.ctx, normalized)
|
||||
if err == nil {
|
||||
r.canReadRenormalized = true
|
||||
}
|
||||
}
|
||||
|
||||
func (r *results) checkStringPositions(k, s string) {
|
||||
fs.Infof(r.f, "Writing position file 0x%0X", s)
|
||||
positionError := internal.PositionNone
|
||||
res := internal.ControlResult{
|
||||
Text: s,
|
||||
WriteError: make(map[internal.Position]string, 3),
|
||||
GetError: make(map[internal.Position]string, 3),
|
||||
InList: make(map[internal.Position]internal.Presence, 3),
|
||||
}
|
||||
|
||||
for _, pos := range internal.PositionList {
|
||||
path := ""
|
||||
switch pos {
|
||||
case internal.PositionMiddle:
|
||||
path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
|
||||
case internal.PositionLeft:
|
||||
path = fmt.Sprintf("%s-position-left-%0X", s, s)
|
||||
case internal.PositionRight:
|
||||
path = fmt.Sprintf("position-right-%0X-%s", s, s)
|
||||
default:
|
||||
panic("invalid position: " + pos.String())
|
||||
}
|
||||
_, writeError := r.writeFile(path)
|
||||
if writeError != nil {
|
||||
res.WriteError[pos] = writeError.Error()
|
||||
fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeError)
|
||||
} else {
|
||||
fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
|
||||
}
|
||||
obj, getErr := r.f.NewObject(r.ctx, path)
|
||||
if getErr != nil {
|
||||
res.GetError[pos] = getErr.Error()
|
||||
fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
|
||||
} else {
|
||||
if obj.Size() != 50 {
|
||||
res.GetError[pos] = fmt.Sprintf("invalid size %d", obj.Size())
|
||||
fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
|
||||
} else {
|
||||
fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
|
||||
}
|
||||
}
|
||||
if writeError != nil || getErr != nil {
|
||||
positionError += pos
|
||||
}
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
r.stringNeedsEscaping[k] = positionError
|
||||
r.controlResults[k] = res
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// check we can write a file with the control chars
|
||||
func (r *results) checkControls() {
|
||||
fs.Infof(r.f, "Trying to create control character file names")
|
||||
ci := fs.GetConfig(context.Background())
|
||||
|
||||
// Concurrency control
|
||||
tokens := make(chan struct{}, ci.Checkers)
|
||||
for i := 0; i < ci.Checkers; i++ {
|
||||
tokens <- struct{}{}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := rune(0); i < 128; i++ {
|
||||
s := string(i)
|
||||
if i == 0 || i == '/' {
|
||||
// We're not even going to check NULL or /
|
||||
r.stringNeedsEscaping[s] = internal.PositionAll
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(s string) {
|
||||
defer wg.Done()
|
||||
token := <-tokens
|
||||
k := s
|
||||
r.checkStringPositions(k, s)
|
||||
tokens <- token
|
||||
}(s)
|
||||
}
|
||||
for _, s := range []string{"＼", "\u00A0", "\xBF", "\xFE"} {
|
||||
wg.Add(1)
|
||||
go func(s string) {
|
||||
defer wg.Done()
|
||||
token := <-tokens
|
||||
k := s
|
||||
r.checkStringPositions(k, s)
|
||||
tokens <- token
|
||||
}(s)
|
||||
}
|
||||
wg.Wait()
|
||||
r.checkControlsList()
|
||||
fs.Infof(r.f, "Done trying to create control character file names")
|
||||
}
|
||||
|
||||
func (r *results) checkControlsList() {
|
||||
l, err := r.f.List(context.TODO(), "")
|
||||
if err != nil {
|
||||
fs.Errorf(r.f, "Listing control character file names failed: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
namesMap := make(map[string]struct{}, len(l))
|
||||
for _, s := range l {
|
||||
namesMap[path.Base(s.Remote())] = struct{}{}
|
||||
}
|
||||
|
||||
for path := range namesMap {
|
||||
var pos internal.Position
|
||||
var hex, value string
|
||||
if g := positionLeftRe.FindStringSubmatch(path); g != nil {
|
||||
pos, hex, value = internal.PositionLeft, g[2], g[1]
|
||||
} else if g := positionMiddleRe.FindStringSubmatch(path); g != nil {
|
||||
pos, hex, value = internal.PositionMiddle, g[1], g[2]
|
||||
} else if g := positionRightRe.FindStringSubmatch(path); g != nil {
|
||||
pos, hex, value = internal.PositionRight, g[1], g[2]
|
||||
} else {
|
||||
fs.Infof(r.f, "Unknown path %q", path)
|
||||
continue
|
||||
}
|
||||
var hexValue []byte
|
||||
for ; len(hex) >= 2; hex = hex[2:] {
|
||||
if b, err := strconv.ParseUint(hex[:2], 16, 8); err != nil {
|
||||
fs.Infof(r.f, "Invalid path %q: %s", path, err)
|
||||
continue
|
||||
} else {
|
||||
hexValue = append(hexValue, byte(b))
|
||||
}
|
||||
}
|
||||
if hex != "" {
|
||||
fs.Infof(r.f, "Invalid path %q", path)
|
||||
continue
|
||||
}
|
||||
|
||||
hexStr := string(hexValue)
|
||||
k := hexStr
|
||||
switch r.controlResults[k].InList[pos] {
|
||||
case internal.Absent:
|
||||
if hexStr == value {
|
||||
r.controlResults[k].InList[pos] = internal.Present
|
||||
} else {
|
||||
r.controlResults[k].InList[pos] = internal.Renamed
|
||||
}
|
||||
case internal.Present:
|
||||
r.controlResults[k].InList[pos] = internal.Multiple
|
||||
case internal.Renamed:
|
||||
r.controlResults[k].InList[pos] = internal.Multiple
|
||||
}
|
||||
delete(namesMap, path)
|
||||
}
|
||||
|
||||
if len(namesMap) > 0 {
|
||||
fs.Infof(r.f, "Found additional control character file names:")
|
||||
for name := range namesMap {
|
||||
fs.Infof(r.f, "%q", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// find the max file name size we can use
|
||||
func (r *results) findMaxLength() {
|
||||
const maxLen = 16 * 1024
|
||||
name := make([]byte, maxLen)
|
||||
for i := range name {
|
||||
name[i] = 'a'
|
||||
}
|
||||
// Find the first size of filename we can't write
|
||||
i := sort.Search(len(name), func(i int) (fail bool) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
||||
fail = true
|
||||
}
|
||||
}()
|
||||
|
||||
path := string(name[:i])
|
||||
_, err := r.writeFile(path)
|
||||
if err != nil {
|
||||
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
||||
return true
|
||||
}
|
||||
fs.Infof(r.f, "Wrote file with name length %d", i)
|
||||
return false
|
||||
})
|
||||
r.maxFileLength = i - 1
|
||||
fs.Infof(r.f, "Max file length is %d", r.maxFileLength)
|
||||
}
|
||||
|
||||
func (r *results) checkStreaming() {
|
||||
putter := r.f.Put
|
||||
if r.f.Features().PutStream != nil {
|
||||
fs.Infof(r.f, "Given remote has specialized streaming function. Using that to test streaming.")
|
||||
putter = r.f.Features().PutStream
|
||||
}
|
||||
|
||||
contents := "thinking of test strings is hard"
|
||||
buf := bytes.NewBufferString(contents)
|
||||
hashIn := hash.NewMultiHasher()
|
||||
in := io.TeeReader(buf, hashIn)
|
||||
|
||||
objIn := object.NewStaticObjectInfo("checkStreamingTest", time.Now(), -1, true, nil, r.f)
|
||||
objR, err := putter(r.ctx, in, objIn)
|
||||
if err != nil {
|
||||
fs.Infof(r.f, "Streamed file failed to upload (%v)", err)
|
||||
r.canStream = false
|
||||
return
|
||||
}
|
||||
|
||||
hashes := hashIn.Sums()
|
||||
types := objR.Fs().Hashes().Array()
|
||||
for _, Hash := range types {
|
||||
sum, err := objR.Hash(r.ctx, Hash)
|
||||
if err != nil {
|
||||
fs.Infof(r.f, "Streamed file failed when getting hash %v (%v)", Hash, err)
|
||||
r.canStream = false
|
||||
return
|
||||
}
|
||||
if !hash.Equals(hashes[Hash], sum) {
|
||||
fs.Infof(r.f, "Streamed file has incorrect hash %v: expecting %q got %q", Hash, hashes[Hash], sum)
|
||||
r.canStream = false
|
||||
return
|
||||
}
|
||||
}
|
||||
if int64(len(contents)) != objR.Size() {
|
||||
fs.Infof(r.f, "Streamed file has incorrect file size: expecting %d got %d", len(contents), objR.Size())
|
||||
r.canStream = false
|
||||
return
|
||||
}
|
||||
r.canStream = true
|
||||
}
|
||||
|
||||
func readInfo(ctx context.Context, f fs.Fs) error {
|
||||
err := f.Mkdir(ctx, "")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't mkdir")
|
||||
}
|
||||
r := newResults(ctx, f)
|
||||
if checkControl {
|
||||
r.checkControls()
|
||||
}
|
||||
if checkLength {
|
||||
r.findMaxLength()
|
||||
}
|
||||
if checkNormalization {
|
||||
r.checkUTF8Normalization()
|
||||
}
|
||||
if checkStreaming {
|
||||
r.checkStreaming()
|
||||
}
|
||||
r.Print()
|
||||
r.WriteJSON()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/rclone/rclone/cmd/test/info/internal"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fOut := flag.String("o", "out.csv", "Output file")
|
||||
flag.Parse()
|
||||
|
||||
args := flag.Args()
|
||||
remotes := make([]internal.InfoReport, 0, len(args))
|
||||
for _, fn := range args {
|
||||
f, err := os.Open(fn)
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to open %q: %s", fn, err)
|
||||
}
|
||||
var remote internal.InfoReport
|
||||
dec := json.NewDecoder(f)
|
||||
err = dec.Decode(&remote)
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to decode %q: %s", fn, err)
|
||||
}
|
||||
if remote.ControlCharacters == nil {
|
||||
log.Printf("Skipping remote %s: no ControlCharacters", remote.Remote)
|
||||
} else {
|
||||
remotes = append(remotes, remote)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
log.Fatalf("Closing %q failed: %s", fn, err)
|
||||
}
|
||||
}
|
||||
|
||||
charsMap := make(map[string]string)
|
||||
var remoteNames []string
|
||||
for _, r := range remotes {
|
||||
remoteNames = append(remoteNames, r.Remote)
|
||||
for k, v := range *r.ControlCharacters {
|
||||
v.Text = k
|
||||
quoted := strconv.Quote(k)
|
||||
charsMap[k] = quoted[1 : len(quoted)-1]
|
||||
}
|
||||
}
|
||||
sort.Strings(remoteNames)
|
||||
|
||||
chars := make([]string, 0, len(charsMap))
|
||||
for k := range charsMap {
|
||||
chars = append(chars, k)
|
||||
}
|
||||
sort.Strings(chars)
|
||||
|
||||
// char remote output
|
||||
recordsMap := make(map[string]map[string][]string)
|
||||
// remote output
|
||||
hRemoteMap := make(map[string][]string)
|
||||
hOperation := []string{"Write", "Write", "Write", "Get", "Get", "Get", "List", "List", "List"}
|
||||
hPosition := []string{"L", "M", "R", "L", "M", "R", "L", "M", "R"}
|
||||
|
||||
// remote
|
||||
// write get list
|
||||
// left middle right left middle right left middle right
|
||||
|
||||
for _, r := range remotes {
|
||||
hRemoteMap[r.Remote] = []string{r.Remote, "", "", "", "", "", "", "", ""}
|
||||
for k, v := range *r.ControlCharacters {
|
||||
cMap, ok := recordsMap[k]
|
||||
if !ok {
|
||||
cMap = make(map[string][]string, 1)
|
||||
recordsMap[k] = cMap
|
||||
}
|
||||
|
||||
cMap[r.Remote] = []string{
|
||||
sok(v.WriteError[internal.PositionLeft]), sok(v.WriteError[internal.PositionMiddle]), sok(v.WriteError[internal.PositionRight]),
|
||||
sok(v.GetError[internal.PositionLeft]), sok(v.GetError[internal.PositionMiddle]), sok(v.GetError[internal.PositionRight]),
|
||||
pok(v.InList[internal.PositionLeft]), pok(v.InList[internal.PositionMiddle]), pok(v.InList[internal.PositionRight]),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
records := [][]string{
|
||||
{"", ""},
|
||||
{"", ""},
|
||||
{"Bytes", "Char"},
|
||||
}
|
||||
for _, r := range remoteNames {
|
||||
records[0] = append(records[0], hRemoteMap[r]...)
|
||||
records[1] = append(records[1], hOperation...)
|
||||
records[2] = append(records[2], hPosition...)
|
||||
}
|
||||
for _, c := range chars {
|
||||
k := charsMap[c]
|
||||
row := []string{fmt.Sprintf("%X", c), k}
|
||||
for _, r := range remoteNames {
|
||||
if m, ok := recordsMap[c][r]; ok {
|
||||
row = append(row, m...)
|
||||
} else {
|
||||
row = append(row, "", "", "", "", "", "", "", "", "")
|
||||
}
|
||||
}
|
||||
records = append(records, row)
|
||||
}
|
||||
|
||||
var writer io.Writer
|
||||
if *fOut == "-" {
|
||||
writer = os.Stdout
|
||||
} else {
|
||||
f, err := os.Create(*fOut)
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to create %q: %s", *fOut, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := f.Close(); err != nil {
|
||||
log.Fatalln("Error writing csv:", err)
|
||||
}
|
||||
}()
|
||||
writer = f
|
||||
}
|
||||
|
||||
w := csv.NewWriter(writer)
|
||||
err := w.WriteAll(records)
|
||||
if err != nil {
|
||||
log.Fatalln("Error writing csv:", err)
|
||||
} else if err := w.Error(); err != nil {
|
||||
log.Fatalln("Error writing csv:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func sok(s string) string {
|
||||
if s != "" {
|
||||
return "ERR"
|
||||
}
|
||||
return "OK"
|
||||
}
|
||||
|
||||
func pok(p internal.Presence) string {
|
||||
switch p {
|
||||
case internal.Absent:
|
||||
return "MIS"
|
||||
case internal.Present:
|
||||
return "OK"
|
||||
case internal.Renamed:
|
||||
return "REN"
|
||||
case internal.Multiple:
|
||||
return "MUL"
|
||||
default:
|
||||
return "ERR"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Presence describes the presence of a filename in file listing
|
||||
type Presence int
|
||||
|
||||
// Possible Presence states
|
||||
const (
|
||||
Absent Presence = iota
|
||||
Present
|
||||
Renamed
|
||||
Multiple
|
||||
)
|
||||
|
||||
// Position is the placement of the test character in the filename
|
||||
type Position int
|
||||
|
||||
// Predefined positions
|
||||
const (
|
||||
PositionMiddle Position = 1 << iota
|
||||
PositionLeft
|
||||
PositionRight
|
||||
PositionNone Position = 0
|
||||
PositionAll Position = PositionRight<<1 - 1
|
||||
)
|
||||
|
||||
// PositionList contains all valid positions
|
||||
var PositionList = []Position{PositionMiddle, PositionLeft, PositionRight}
|
||||
|
||||
// ControlResult contains the result of a single character test
|
||||
type ControlResult struct {
|
||||
Text string `json:"-"`
|
||||
WriteError map[Position]string
|
||||
GetError map[Position]string
|
||||
InList map[Position]Presence
|
||||
}
|
||||
|
||||
// InfoReport is the structure of the JSON output
|
||||
type InfoReport struct {
|
||||
Remote string
|
||||
ControlCharacters *map[string]ControlResult
|
||||
MaxFileLength *int
|
||||
CanStream *bool
|
||||
CanWriteUnnormalized *bool
|
||||
CanReadUnnormalized *bool
|
||||
CanReadRenormalized *bool
|
||||
}
|
||||
|
||||
func (e Position) String() string {
|
||||
switch e {
|
||||
case PositionNone:
|
||||
return "none"
|
||||
case PositionAll:
|
||||
return "all"
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if e&PositionMiddle != 0 {
|
||||
buf.WriteString("middle")
|
||||
e &= ^PositionMiddle
|
||||
}
|
||||
if e&PositionLeft != 0 {
|
||||
if buf.Len() != 0 {
|
||||
buf.WriteRune(',')
|
||||
}
|
||||
buf.WriteString("left")
|
||||
e &= ^PositionLeft
|
||||
}
|
||||
if e&PositionRight != 0 {
|
||||
if buf.Len() != 0 {
|
||||
buf.WriteRune(',')
|
||||
}
|
||||
buf.WriteString("right")
|
||||
e &= ^PositionRight
|
||||
}
|
||||
if e != PositionNone {
|
||||
panic("invalid position")
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// MarshalText encodes the position when used as a map key
|
||||
func (e Position) MarshalText() ([]byte, error) {
|
||||
return []byte(e.String()), nil
|
||||
}
|
||||
|
||||
// UnmarshalText decodes a position when used as a map key
|
||||
func (e *Position) UnmarshalText(text []byte) error {
|
||||
switch s := strings.ToLower(string(text)); s {
|
||||
default:
|
||||
*e = PositionNone
|
||||
for _, p := range strings.Split(s, ",") {
|
||||
switch p {
|
||||
case "left":
|
||||
*e |= PositionLeft
|
||||
case "middle":
|
||||
*e |= PositionMiddle
|
||||
case "right":
|
||||
*e |= PositionRight
|
||||
default:
|
||||
return fmt.Errorf("unknown position: %s", e)
|
||||
}
|
||||
}
|
||||
case "none":
|
||||
*e = PositionNone
|
||||
case "all":
|
||||
*e = PositionAll
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e Presence) String() string {
|
||||
switch e {
|
||||
case Absent:
|
||||
return "absent"
|
||||
case Present:
|
||||
return "present"
|
||||
case Renamed:
|
||||
return "renamed"
|
||||
case Multiple:
|
||||
return "multiple"
|
||||
default:
|
||||
panic("invalid presence")
|
||||
}
|
||||
}
|
||||
|
||||
// MarshalJSON encodes the presence when used as a JSON value
|
||||
func (e Presence) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(e.String())
|
||||
}
|
||||
|
||||
// UnmarshalJSON decodes a presence when used as a JSON value
|
||||
func (e *Presence) UnmarshalJSON(text []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(text, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
switch s := strings.ToLower(s); s {
|
||||
case "absent":
|
||||
*e = Absent
|
||||
case "present":
|
||||
*e = Present
|
||||
case "renamed":
|
||||
*e = Renamed
|
||||
case "multiple":
|
||||
*e = Multiple
|
||||
default:
|
||||
return fmt.Errorf("unknown presence: %s", e)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
set RCLONE_CONFIG_LOCALWINDOWS_TYPE=local
|
||||
rclone.exe purge LocalWindows:info
|
||||
rclone.exe info -vv LocalWindows:info --write-json=info-LocalWindows.json > info-LocalWindows.log 2>&1
|
||||
rclone.exe ls -vv LocalWindows:info > info-LocalWindows.list 2>&1
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env zsh
|
||||
#
|
||||
# example usage:
|
||||
# $GOPATH/src/github.com/rclone/rclone/cmd/info/test.sh --list | \
|
||||
# parallel -P20 $GOPATH/src/github.com/rclone/rclone/cmd/info/test.sh
|
||||
|
||||
export PATH=$GOPATH/src/github.com/rclone/rclone:$PATH
|
||||
|
||||
typeset -A allRemotes
|
||||
allRemotes=(
|
||||
TestAmazonCloudDrive '--low-level-retries=2 --checkers=5 --upload-wait=5s'
|
||||
TestB2 ''
|
||||
TestBox ''
|
||||
TestDrive '--tpslimit=5'
|
||||
TestCrypt ''
|
||||
TestDropbox '--checkers=1'
|
||||
TestGCS ''
|
||||
TestJottacloud ''
|
||||
TestKoofr ''
|
||||
TestMega ''
|
||||
TestOneDrive ''
|
||||
TestOpenDrive '--low-level-retries=4 --checkers=5'
|
||||
TestPcloud '--low-level-retries=2 --timeout=15s'
|
||||
TestS3 ''
|
||||
Local ''
|
||||
)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
set -- ${(k)allRemotes[@]}
|
||||
elif [[ $1 = --list ]]; then
|
||||
printf '%s\n' ${(k)allRemotes[@]}
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for remote; do
|
||||
case $remote in
|
||||
Local)
|
||||
l=Local$(uname)
|
||||
export RCLONE_CONFIG_${l:u}_TYPE=local
|
||||
dir=$l:infotest;;
|
||||
TestGCS)
|
||||
dir=$remote:$GCS_BUCKET/infotest;;
|
||||
*)
|
||||
dir=$remote:infotest;;
|
||||
esac
|
||||
|
||||
rclone purge $dir || :
|
||||
rclone info -vv $dir --write-json=info-$remote.json ${=allRemotes[$remote]:-} &> info-$remote.log
|
||||
rclone ls -vv $dir &> info-$remote.list
|
||||
done
|
||||
@@ -0,0 +1,52 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/test"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
test.Command.AddCommand(commandDefinition)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "memory remote:path",
|
||||
Short: `Load all the objects at remote:path into memory and report memory stats.`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
fsrc := cmd.NewFsSrc(args)
|
||||
cmd.Run(false, false, command, func() error {
|
||||
ctx := context.Background()
|
||||
objects, _, err := operations.Count(ctx, fsrc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
objs := make([]fs.Object, 0, objects)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
var mu sync.Mutex
|
||||
err = operations.ListFn(ctx, fsrc, func(o fs.Object) {
|
||||
mu.Lock()
|
||||
objs = append(objs, o)
|
||||
mu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&after)
|
||||
usedMemory := after.Alloc - before.Alloc
|
||||
fs.Logf(nil, "%d objects took %d bytes, %.1f bytes/object", len(objs), usedMemory, float64(usedMemory)/float64(len(objs)))
|
||||
fs.Logf(nil, "System memory changed from %d to %d bytes a change of %d bytes", before.Sys, after.Sys, after.Sys-before.Sys)
|
||||
return nil
|
||||
})
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(Command)
|
||||
}
|
||||
|
||||
// Command definition for cobra
|
||||
var Command = &cobra.Command{
|
||||
Use: "test <subcommand>",
|
||||
Short: `Run a test command`,
|
||||
Long: `Rclone test is used to run test commands.
|
||||
|
||||
Select which test comand you want with the subcommand, eg
|
||||
|
||||
rclone test memory remote:
|
||||
|
||||
Each subcommand has its own options which you can see in their help.
|
||||
|
||||
**NB** Be careful running these commands, they may do strange things
|
||||
so reading their documentation first is recommended.
|
||||
`,
|
||||
}
|
||||
Reference in New Issue
Block a user