f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add options to Put, PutUnchecked and Update, add HashOption and speed up local

Browse Source 
* Add options to Put, PutUnchecked and Update for all Fses
  * Use these to create HashOption
  * Implement this in local
  * Pass the option in fs.Copy

This has the effect that we only calculate hashes we need to in the
local Fs which speeds up transfers significantly.
This commit is contained in:
Nick Craig-Wood
2017-05-29 12:04:52 +01:00
parent 6381959850
commit 20da3e6352
17 changed files with 101 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/operations.go
+15 -9
View File
@@ -268,6 +268,17 @@ func Copy(f Fs, dst Object, remote string, src Object) (err error) {
maxTries := Config.LowLevelRetries
tries := 0
doUpdate := dst != nil
// work out which hash to use - limit to 1 hash in common
var common HashSet
hashType := HashNone
if !Config.SizeOnly {
common = src.Fs().Hashes().Overlap(f.Hashes())
if common.Count() > 0 {
hashType = common.GetOne()
common = HashSet(hashType)
}
}
hashOption := &HashesOption{Hashes: common}
var actionTaken string
for {
// Try server side copy first - if has optional interface and
@@ -285,7 +296,7 @@ func Copy(f Fs, dst Object, remote string, src Object) (err error) {
// If can't server side copy, do it manually
if err == ErrorCantCopy {
var in0 io.ReadCloser
in0, err = src.Open()
in0, err = src.Open(hashOption)
if err != nil {
err = errors.Wrap(err, "failed to open source object")
} else {
@@ -297,10 +308,10 @@ func Copy(f Fs, dst Object, remote string, src Object) (err error) {
}
if doUpdate {
actionTaken = "Copied (replaced existing)"
err = dst.Update(in, wrappedSrc)
err = dst.Update(in, wrappedSrc, hashOption)
} else {
actionTaken = "Copied (new)"
dst, err = f.Put(in, wrappedSrc)
dst, err = f.Put(in, wrappedSrc, hashOption)
}
closeErr := in.Close()
if err == nil {
@@ -338,12 +349,7 @@ func Copy(f Fs, dst Object, remote string, src Object) (err error) {
// Verify hashes are the same after transfer - ignoring blank hashes
// TODO(klauspost): This could be extended, so we always create a hash type matching
// the destination, and calculate it while sending.
common := src.Fs().Hashes().Overlap(dst.Fs().Hashes())
// Debugf(src, "common hashes: %v", common)
if !Config.SizeOnly && common.Count() > 0 {
// Get common hash type
hashType := common.GetOne()
if hashType != HashNone {
var srcSum string
srcSum, err = src.Hash(hashType)
if err != nil {