Teddy TennantandNick Craig-Wood
a182fcbce8
rclone checksum and rclone check --checkfile hashed one file at a time unless --download was in use. The non-download branch of CheckSum called Object.Hash synchronously from inside the listing callback, so --checkers had no effect there; only the --download branch used the checker token pool. Both branches now do their work in the same goroutine bounded by --checkers, so --checkers means the same thing with and without --download. Checking a sum file against 2.6 GiB of local files with --checkers 8 went from 1.70s to 0.31s here. As a result the "hashing" checking transfer now covers the hashing work in both branches, rather than being finished as soon as the download goroutine had been started.
736 lines
21 KiB
Go
736 lines
21 KiB
Go
package operations_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/cmd/bisync/bilib"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/accounting"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/fstest/mockfs"
|
|
"github.com/rclone/rclone/fstest/mockobject"
|
|
"github.com/rclone/rclone/lib/readers"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/text/unicode/norm"
|
|
)
|
|
|
|
func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) {
|
|
r := fstest.NewRun(t)
|
|
ctx := context.Background()
|
|
ci := fs.GetConfig(ctx)
|
|
|
|
addBuffers := func(opt *operations.CheckOpt) {
|
|
opt.Combined = new(bytes.Buffer)
|
|
opt.MissingOnSrc = new(bytes.Buffer)
|
|
opt.MissingOnDst = new(bytes.Buffer)
|
|
opt.Match = new(bytes.Buffer)
|
|
opt.Differ = new(bytes.Buffer)
|
|
opt.Error = new(bytes.Buffer)
|
|
}
|
|
|
|
sortLines := func(in string) []string {
|
|
if in == "" {
|
|
return []string{}
|
|
}
|
|
lines := strings.Split(in, "\n")
|
|
sort.Strings(lines)
|
|
return lines
|
|
}
|
|
|
|
checkBuffer := func(name string, want map[string]string, out io.Writer) {
|
|
expected := want[name]
|
|
buf, ok := out.(*bytes.Buffer)
|
|
require.True(t, ok)
|
|
assert.Equal(t, sortLines(expected), sortLines(buf.String()), name)
|
|
}
|
|
|
|
checkBuffers := func(opt *operations.CheckOpt, want map[string]string) {
|
|
checkBuffer("combined", want, opt.Combined)
|
|
checkBuffer("missingonsrc", want, opt.MissingOnSrc)
|
|
checkBuffer("missingondst", want, opt.MissingOnDst)
|
|
checkBuffer("match", want, opt.Match)
|
|
checkBuffer("differ", want, opt.Differ)
|
|
checkBuffer("error", want, opt.Error)
|
|
}
|
|
|
|
check := func(i int, wantErrors int64, wantChecks int64, oneway bool, wantOutput map[string]string) {
|
|
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
|
|
accounting.GlobalStats().ResetCounters()
|
|
opt := operations.CheckOpt{
|
|
Fdst: r.Fremote,
|
|
Fsrc: r.Flocal,
|
|
OneWay: oneway,
|
|
}
|
|
addBuffers(&opt)
|
|
var err error
|
|
buf := bilib.CaptureOutput(func() {
|
|
err = checkFunction(ctx, &opt)
|
|
})
|
|
gotErrors := accounting.GlobalStats().GetErrors()
|
|
gotChecks := accounting.GlobalStats().GetChecks()
|
|
if wantErrors == 0 && err != nil {
|
|
t.Errorf("%d: Got error when not expecting one: %v", i, err)
|
|
}
|
|
if wantErrors != 0 && err == nil {
|
|
t.Errorf("%d: No error when expecting one", i)
|
|
}
|
|
if wantErrors != gotErrors {
|
|
t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
|
|
}
|
|
if gotChecks > 0 && !strings.Contains(string(buf), "matching files") {
|
|
t.Errorf("%d: Total files matching line missing", i)
|
|
}
|
|
if wantChecks != gotChecks {
|
|
t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
|
|
}
|
|
checkBuffers(&opt, wantOutput)
|
|
})
|
|
}
|
|
|
|
file1 := r.WriteBoth(ctx, "rutabaga", "is tasty", t3)
|
|
r.CheckRemoteItems(t, file1)
|
|
r.CheckLocalItems(t, file1)
|
|
check(1, 0, 1, false, map[string]string{
|
|
"combined": "= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "rutabaga\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
|
|
r.CheckLocalItems(t, file1, file2)
|
|
check(2, 1, 1, false, map[string]string{
|
|
"combined": "+ potato2\n= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "potato2\n",
|
|
"match": "rutabaga\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file3 := r.WriteObject(ctx, "empty space", "-", t2)
|
|
r.CheckRemoteItems(t, file1, file3)
|
|
check(3, 2, 1, false, map[string]string{
|
|
"combined": "- empty space\n+ potato2\n= rutabaga\n",
|
|
"missingonsrc": "empty space\n",
|
|
"missingondst": "potato2\n",
|
|
"match": "rutabaga\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file2r := file2
|
|
if ci.SizeOnly {
|
|
file2r = r.WriteObject(ctx, "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
|
|
} else {
|
|
r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1)
|
|
}
|
|
r.CheckRemoteItems(t, file1, file2r, file3)
|
|
check(4, 1, 2, false, map[string]string{
|
|
"combined": "- empty space\n= potato2\n= rutabaga\n",
|
|
"missingonsrc": "empty space\n",
|
|
"missingondst": "",
|
|
"match": "rutabaga\npotato2\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file3r := file3
|
|
file3l := r.WriteFile("empty space", "DIFFER", t2)
|
|
r.CheckLocalItems(t, file1, file2, file3l)
|
|
check(5, 1, 3, false, map[string]string{
|
|
"combined": "* empty space\n= potato2\n= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "potato2\nrutabaga\n",
|
|
"differ": "empty space\n",
|
|
"error": "",
|
|
})
|
|
|
|
file4 := r.WriteObject(ctx, "remotepotato", "------------------------------------------------------------", t1)
|
|
r.CheckRemoteItems(t, file1, file2r, file3r, file4)
|
|
check(6, 2, 3, false, map[string]string{
|
|
"combined": "* empty space\n= potato2\n= rutabaga\n- remotepotato\n",
|
|
"missingonsrc": "remotepotato\n",
|
|
"missingondst": "",
|
|
"match": "potato2\nrutabaga\n",
|
|
"differ": "empty space\n",
|
|
"error": "",
|
|
})
|
|
check(7, 1, 3, true, map[string]string{
|
|
"combined": "* empty space\n= potato2\n= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "potato2\nrutabaga\n",
|
|
"differ": "empty space\n",
|
|
"error": "",
|
|
})
|
|
}
|
|
|
|
func TestCheck(t *testing.T) {
|
|
testCheck(t, operations.Check)
|
|
}
|
|
|
|
func TestCheckFsError(t *testing.T) {
|
|
ctx := context.Background()
|
|
dstFs, err := fs.NewFs(ctx, "nonexistent")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srcFs, err := fs.NewFs(ctx, "nonexistent")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opt := operations.CheckOpt{
|
|
Fdst: dstFs,
|
|
Fsrc: srcFs,
|
|
OneWay: false,
|
|
}
|
|
err = operations.Check(ctx, &opt)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestCheckDownload(t *testing.T) {
|
|
testCheck(t, operations.CheckDownload)
|
|
}
|
|
|
|
func TestCheckSizeOnly(t *testing.T) {
|
|
ctx := context.Background()
|
|
ci := fs.GetConfig(ctx)
|
|
ci.SizeOnly = true
|
|
defer func() { ci.SizeOnly = false }()
|
|
TestCheck(t)
|
|
}
|
|
|
|
func TestCheckEqualReaders(t *testing.T) {
|
|
b65a := make([]byte, 65*1024)
|
|
b65b := make([]byte, 65*1024)
|
|
b65b[len(b65b)-1] = 1
|
|
b66 := make([]byte, 66*1024)
|
|
|
|
equal, err := operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65a))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, equal, true)
|
|
|
|
equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65b))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b66))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), bytes.NewBuffer(b65a))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
myErr := errors.New("sentinel")
|
|
wrap := func(b []byte) io.Reader {
|
|
r := bytes.NewBuffer(b)
|
|
e := readers.ErrorReader{Err: myErr}
|
|
return io.MultiReader(r, e)
|
|
}
|
|
|
|
equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65b))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b66))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(wrap(b66), bytes.NewBuffer(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65b))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b66))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
|
|
equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), wrap(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, equal, false)
|
|
}
|
|
|
|
func TestParseSumFile(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
ctx := context.Background()
|
|
|
|
const sumFile = "test.sum"
|
|
|
|
samples := []struct {
|
|
hash, sep, name string
|
|
ok bool
|
|
}{
|
|
{"1", " ", "file1", true},
|
|
{"2", " *", "file2", true},
|
|
{"3", " ", " file3 ", true},
|
|
{"4", " ", "\tfile3\t", true},
|
|
{"5", " ", "file5", false},
|
|
{"6", "\t", "file6", false},
|
|
{"7", " \t", " file7 ", false},
|
|
{"", " ", "file8", false},
|
|
{"", "", "file9", false},
|
|
}
|
|
|
|
for _, eol := range []string{"\n", "\r\n"} {
|
|
data := &bytes.Buffer{}
|
|
wantNum := 0
|
|
for _, s := range samples {
|
|
_, _ = data.WriteString(s.hash + s.sep + s.name + eol)
|
|
if s.ok {
|
|
wantNum++
|
|
}
|
|
}
|
|
|
|
_ = r.WriteObject(ctx, sumFile, data.String(), t1)
|
|
file := fstest.NewObject(ctx, t, r.Fremote, sumFile)
|
|
sums, err := operations.ParseSumFile(ctx, file)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, wantNum, len(sums))
|
|
for _, s := range samples {
|
|
if s.ok {
|
|
assert.Equal(t, s.hash, sums[s.name])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func testCheckSum(t *testing.T, download bool) {
|
|
const dataDir = "data"
|
|
const sumFile = "test.sum"
|
|
|
|
hashType := hash.MD5
|
|
const (
|
|
testString1 = "Hello, World!"
|
|
testDigest1 = "65a8e27d8879283831b664bd8b7f0ad4"
|
|
testDigest1Upper = "65A8E27D8879283831B664BD8B7F0AD4"
|
|
testString2 = "I am the walrus"
|
|
testDigest2 = "87396e030ef3f5b35bbf85c0a09a4fb3"
|
|
testDigest2Mixed = "87396e030EF3f5b35BBf85c0a09a4FB3"
|
|
)
|
|
|
|
type wantType map[string]string
|
|
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
|
|
subRemote := r.FremoteName
|
|
if !strings.HasSuffix(subRemote, ":") {
|
|
subRemote += "/"
|
|
}
|
|
subRemote += dataDir
|
|
dataFs, err := fs.NewFs(ctx, subRemote)
|
|
require.NoError(t, err)
|
|
|
|
if !download && !dataFs.Hashes().Contains(hashType) {
|
|
t.Skipf("%s lacks %s, skipping", dataFs, hashType)
|
|
}
|
|
|
|
makeFile := func(name, content string) fstest.Item {
|
|
remote := dataDir + "/" + name
|
|
return r.WriteObject(ctx, remote, content, t1)
|
|
}
|
|
|
|
makeSums := func(sums operations.HashSums) fstest.Item {
|
|
files := make([]string, 0, len(sums))
|
|
for name := range sums {
|
|
files = append(files, name)
|
|
}
|
|
sort.Strings(files)
|
|
buf := &bytes.Buffer{}
|
|
for _, name := range files {
|
|
_, _ = fmt.Fprintf(buf, "%s %s\n", sums[name], name)
|
|
}
|
|
return r.WriteObject(ctx, sumFile, buf.String(), t1)
|
|
}
|
|
|
|
sortLines := func(in string) []string {
|
|
if in == "" {
|
|
return []string{}
|
|
}
|
|
lines := strings.Split(in, "\n")
|
|
sort.Strings(lines)
|
|
return lines
|
|
}
|
|
|
|
checkResult := func(runNo int, want wantType, name string, out io.Writer) {
|
|
expected := want[name]
|
|
buf, ok := out.(*bytes.Buffer)
|
|
require.True(t, ok)
|
|
assert.Equal(t, sortLines(expected), sortLines(buf.String()), "wrong %s result in run %d", name, runNo)
|
|
}
|
|
|
|
checkRun := func(runNo, wantChecks, wantErrors int, want wantType) {
|
|
accounting.GlobalStats().ResetCounters()
|
|
|
|
opt := operations.CheckOpt{
|
|
Combined: new(bytes.Buffer),
|
|
Match: new(bytes.Buffer),
|
|
Differ: new(bytes.Buffer),
|
|
Error: new(bytes.Buffer),
|
|
MissingOnSrc: new(bytes.Buffer),
|
|
MissingOnDst: new(bytes.Buffer),
|
|
}
|
|
var err error
|
|
buf := bilib.CaptureOutput(func() {
|
|
err = operations.CheckSum(ctx, dataFs, r.Fremote, sumFile, hashType, &opt, download)
|
|
})
|
|
gotErrors := int(accounting.GlobalStats().GetErrors())
|
|
if wantErrors == 0 {
|
|
assert.NoError(t, err, "unexpected error in run %d", runNo)
|
|
}
|
|
if wantErrors > 0 {
|
|
assert.Error(t, err, "no expected error in run %d", runNo)
|
|
}
|
|
assert.Equal(t, wantErrors, gotErrors, "wrong error count in run %d", runNo)
|
|
|
|
gotChecks := int(accounting.GlobalStats().GetChecks())
|
|
if wantChecks > 0 || gotChecks > 0 {
|
|
assert.Contains(t, string(buf), "matching files", "missing matching files in run %d", runNo)
|
|
}
|
|
assert.Equal(t, wantChecks, gotChecks, "wrong number of checks in run %d", runNo)
|
|
|
|
checkResult(runNo, want, "combined", opt.Combined)
|
|
checkResult(runNo, want, "missingonsrc", opt.MissingOnSrc)
|
|
checkResult(runNo, want, "missingondst", opt.MissingOnDst)
|
|
checkResult(runNo, want, "match", opt.Match)
|
|
checkResult(runNo, want, "differ", opt.Differ)
|
|
checkResult(runNo, want, "error", opt.Error)
|
|
}
|
|
|
|
check := func(runNo, wantChecks, wantErrors int, wantResults wantType) {
|
|
runName := fmt.Sprintf("subtest%d", runNo)
|
|
t.Run(runName, func(t *testing.T) {
|
|
checkRun(runNo, wantChecks, wantErrors, wantResults)
|
|
})
|
|
}
|
|
|
|
file1 := makeFile("banana", testString1)
|
|
fcsums := makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1)
|
|
check(1, 1, 0, wantType{
|
|
"combined": "= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "banana\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file2 := makeFile("potato", testString2)
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(2, 2, 1, wantType{
|
|
"combined": "- potato\n= banana\n",
|
|
"missingonsrc": "potato\n",
|
|
"missingondst": "",
|
|
"match": "banana\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
"potato": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(3, 2, 0, wantType{
|
|
"combined": "= potato\n= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "banana\npotato\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest2,
|
|
"potato": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(4, 2, 1, wantType{
|
|
"combined": "* banana\n= potato\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "potato\n",
|
|
"differ": "banana\n",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
"potato": testDigest2,
|
|
"orange": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(5, 2, 1, wantType{
|
|
"combined": "+ orange\n= potato\n= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "orange\n",
|
|
"match": "banana\npotato\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
"potato": testDigest1,
|
|
"orange": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(6, 2, 2, wantType{
|
|
"combined": "+ orange\n* potato\n= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "orange\n",
|
|
"match": "banana\n",
|
|
"differ": "potato\n",
|
|
"error": "",
|
|
})
|
|
|
|
// test mixed-case checksums
|
|
file1 = makeFile("banana", testString1)
|
|
file2 = makeFile("potato", testString2)
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1Upper,
|
|
"potato": testDigest2Mixed,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(7, 2, 0, wantType{
|
|
"combined": "= banana\n= potato\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "banana\npotato\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
}
|
|
|
|
func TestCheckSum(t *testing.T) {
|
|
testCheckSum(t, false)
|
|
}
|
|
|
|
func TestCheckSumDownload(t *testing.T) {
|
|
testCheckSum(t, true)
|
|
}
|
|
|
|
// hashProbe records how many hashing operations are running at once. Each
|
|
// operation blocks until want of them are in flight, or until the deadline
|
|
// expires so a serial implementation finishes instead of deadlocking.
|
|
type hashProbe struct {
|
|
mu sync.Mutex
|
|
inFlight int
|
|
max int
|
|
want int
|
|
once sync.Once
|
|
reached chan struct{}
|
|
timedOut chan struct{}
|
|
}
|
|
|
|
func newHashProbe(t *testing.T, want int, deadline time.Duration) *hashProbe {
|
|
p := &hashProbe{
|
|
want: want,
|
|
reached: make(chan struct{}),
|
|
timedOut: make(chan struct{}),
|
|
}
|
|
timer := time.AfterFunc(deadline, func() { close(p.timedOut) })
|
|
t.Cleanup(func() { timer.Stop() })
|
|
return p
|
|
}
|
|
|
|
func (p *hashProbe) enter() {
|
|
p.mu.Lock()
|
|
p.inFlight++
|
|
p.max = max(p.max, p.inFlight)
|
|
full := p.inFlight >= p.want
|
|
p.mu.Unlock()
|
|
if full {
|
|
p.once.Do(func() { close(p.reached) })
|
|
}
|
|
select {
|
|
case <-p.reached:
|
|
case <-p.timedOut:
|
|
}
|
|
}
|
|
|
|
func (p *hashProbe) leave() {
|
|
p.mu.Lock()
|
|
p.inFlight--
|
|
p.mu.Unlock()
|
|
}
|
|
|
|
func (p *hashProbe) maxInFlight() int {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return p.max
|
|
}
|
|
|
|
// probeObject is an object whose hashing is instrumented by a hashProbe
|
|
type probeObject struct {
|
|
*mockobject.ContentMockObject
|
|
probe *hashProbe
|
|
}
|
|
|
|
func (o *probeObject) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
|
o.probe.enter()
|
|
defer o.probe.leave()
|
|
return o.ContentMockObject.Hash(ctx, ht)
|
|
}
|
|
|
|
func (o *probeObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
|
o.probe.enter()
|
|
defer o.probe.leave()
|
|
return o.ContentMockObject.Open(ctx, options...)
|
|
}
|
|
|
|
func testCheckSumConcurrency(t *testing.T, download bool) {
|
|
const (
|
|
checkers = 2
|
|
numFiles = 4
|
|
sumFile = "test.sum"
|
|
)
|
|
hashType := hash.MD5
|
|
|
|
ctx, ci := fs.AddConfig(context.Background())
|
|
ci.Checkers = checkers
|
|
probe := newHashProbe(t, checkers, 10*time.Second)
|
|
|
|
f, err := mockfs.NewFs(ctx, "checkSum", "checkSum", nil)
|
|
require.NoError(t, err)
|
|
fsrc := f.(*mockfs.Fs)
|
|
fsrc.SetHashes(hash.NewHashSet(hashType))
|
|
|
|
sums := &bytes.Buffer{}
|
|
for i := range numFiles {
|
|
remote := fmt.Sprintf("file%d", i)
|
|
o := mockobject.New(remote).WithContent([]byte("contents of "+remote), mockobject.SeekModeNone)
|
|
sum, err := o.Hash(ctx, hashType)
|
|
require.NoError(t, err)
|
|
_, _ = fmt.Fprintf(sums, "%s %s\n", sum, remote)
|
|
fsrc.AddObject(&probeObject{ContentMockObject: o, probe: probe})
|
|
}
|
|
|
|
f, err = mockfs.NewFs(ctx, "sums", "sums", nil)
|
|
require.NoError(t, err)
|
|
fsum := f.(*mockfs.Fs)
|
|
fsum.AddObject(mockobject.New(sumFile).WithContent(sums.Bytes(), mockobject.SeekModeNone))
|
|
|
|
opt := operations.CheckOpt{Combined: new(bytes.Buffer)}
|
|
require.NoError(t, operations.CheckSum(ctx, fsrc, fsum, sumFile, hashType, &opt, download))
|
|
assert.Equal(t, checkers, probe.maxInFlight(), "hashing was not run --checkers at a time")
|
|
}
|
|
|
|
func TestCheckSumConcurrency(t *testing.T) {
|
|
testCheckSumConcurrency(t, false)
|
|
}
|
|
|
|
func TestCheckSumDownloadConcurrency(t *testing.T) {
|
|
testCheckSumConcurrency(t, true)
|
|
}
|
|
|
|
func TestApplyTransforms(t *testing.T) {
|
|
var (
|
|
hashType = hash.MD5
|
|
content = "Hello, World!"
|
|
hash = "65a8e27d8879283831b664bd8b7f0ad4"
|
|
nfc = norm.NFC.String(norm.NFD.String("測試_Русский___ě_áñ"))
|
|
nfd = norm.NFD.String(nfc)
|
|
nfcx2 = nfc + nfc
|
|
nfdx2 = nfd + nfd
|
|
both = nfc + nfd
|
|
upper = "HELLO, WORLD!"
|
|
lower = "hello, world!"
|
|
upperlowermixed = "HeLlO, wOrLd!"
|
|
)
|
|
|
|
testScenario := func(checkfileName, remotefileName, scenario string) {
|
|
r := fstest.NewRunIndividual(t)
|
|
ctx := context.Background()
|
|
ci := fs.GetConfig(ctx)
|
|
opt := operations.CheckOpt{}
|
|
|
|
remotefile := r.WriteObject(ctx, remotefileName, content, t2)
|
|
// test whether remote is capable of running test
|
|
entries, err := r.Fremote.List(ctx, "")
|
|
assert.NoError(t, err)
|
|
if entries.Len() == 1 && entries[0].Remote() != remotefileName {
|
|
t.Skipf("Fs is incapable of running test, skipping: %s (expected: %s (%s) actual: %s (%s))", scenario, remotefileName, detectEncoding(remotefileName), entries[0].Remote(), detectEncoding(entries[0].Remote()))
|
|
}
|
|
|
|
checkfile := r.WriteFile("test.sum", hash+" "+checkfileName, t2)
|
|
r.CheckLocalItems(t, checkfile)
|
|
assert.False(t, checkfileName == remotefile.Path, "Values match but should not: %s %s", checkfileName, remotefile.Path)
|
|
|
|
testname := scenario + " (without normalization)"
|
|
println(testname)
|
|
ci.NoUnicodeNormalization = true
|
|
ci.IgnoreCaseSync = false
|
|
accounting.GlobalStats().ResetCounters()
|
|
err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true)
|
|
assert.Error(t, err, "no expected error for %s %v %v", testname, checkfileName, remotefileName)
|
|
|
|
testname = scenario + " (with normalization)"
|
|
println(testname)
|
|
ci.NoUnicodeNormalization = false
|
|
ci.IgnoreCaseSync = true
|
|
accounting.GlobalStats().ResetCounters()
|
|
err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true)
|
|
assert.NoError(t, err, "unexpected error for %s %v %v", testname, checkfileName, remotefileName)
|
|
}
|
|
|
|
testScenario(upper, lower, "upper checkfile vs. lower remote")
|
|
testScenario(lower, upper, "lower checkfile vs. upper remote")
|
|
testScenario(lower, upperlowermixed, "lower checkfile vs. upperlowermixed remote")
|
|
testScenario(upperlowermixed, upper, "upperlowermixed checkfile vs. upper remote")
|
|
testScenario(nfd, nfc, "NFD checkfile vs. NFC remote")
|
|
testScenario(nfc, nfd, "NFC checkfile vs. NFD remote")
|
|
testScenario(nfdx2, both, "NFDx2 checkfile vs. both remote")
|
|
testScenario(nfcx2, both, "NFCx2 checkfile vs. both remote")
|
|
testScenario(both, nfdx2, "both checkfile vs. NFDx2 remote")
|
|
testScenario(both, nfcx2, "both checkfile vs. NFCx2 remote")
|
|
}
|
|
|
|
func detectEncoding(s string) string {
|
|
if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
|
|
return "BOTH"
|
|
}
|
|
if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
|
|
return "NFD"
|
|
}
|
|
if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) {
|
|
return "NFC"
|
|
}
|
|
return "OTHER"
|
|
}
|