webdav: fix SetModTime failing and hashes missing on Nextcloud

Nextcloud only stores a checksum which is supplied in the OC-Checksum
header of an upload, and discards it again when the modification time
is set with PROPPATCH. Re-sending the checksum in the PROPPATCH (as is
done for ownCloud) is rejected by Nextcloud with 403 Forbidden which
made the whole PROPPATCH fail, so SetModTime returned an error on any
object which had a hash. Uploads from sources without hashes, eg
streamed uploads with `rclone rcat`, were stored with no hash at all.

Use the Nextcloud PATCH extension with the X-Recalculate-Hash header
to have the server calculate and store the SHA1 of an object after a
streamed upload and after setting the modification time. This gives
a server side hash of the stored data which also lets rclone verify
streamed uploads.

backend/webdav/webdav.go

@@ -233,6 +233,7 @@ type Fs struct {
 	ntlmAuthMu         sync.Mutex    // mutex to serialize NTLM auth roundtrips
 	chunksUploadURL    string        // upload URL for nextcloud chunked
 	canChunk           bool          // set if nextcloud and nextcloud_chunk_size is set
+	canRecalcHash      bool          // set if the server can recalculate checksums with PATCH (nextcloud)
 	authSingleflight   *singleflight.Group
 }
 
@@ -664,6 +665,7 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 		f.propsetMtime = true
 		f.hasOCSHA1 = true
 		f.canChunk = true
+		f.canRecalcHash = true
 
 		if f.opt.ChunkSize == 0 {
 			fs.Logf(nil, "Chunked uploads are disabled because nextcloud_chunk_size is set to 0")
@@ -1471,12 +1473,17 @@ var owncloudPropsetWithChecksum = `<?xml version="1.0" encoding="utf-8" ?>
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if o.fs.propsetMtime {
+		// Setting the modification time discards the stored
+		// checksums so they are set again in the same request.
+		// Servers which can't do that recalculate them afterwards.
 		checksums := ""
+		if !o.fs.canRecalcHash {
 			if o.fs.hasOCSHA1 && o.sha1 != "" {
 				checksums = "SHA1:" + o.sha1
 			} else if o.fs.hasOCMD5 && o.md5 != "" {
 				checksums = "MD5:" + o.md5
 			}
+		}
 
 		opts := rest.Opts{
 			Method:     "PROPPATCH",
@@ -1510,6 +1517,12 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 			// in-memory modtime identical to the one a fresh
 			// listing returns
 			o.modTime = modTime.Truncate(time.Second)
+			if o.fs.canRecalcHash && (o.sha1 != "" || o.md5 != "") {
+				err = o.recalculateHash(ctx)
+				if err != nil {
+					return fmt.Errorf("couldn't restore checksum after setting modified time: %w", err)
+				}
+			}
 			return nil
 		}
 		// got an error, but it's possible it actually worked, so double-check
@@ -1608,7 +1621,54 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	// read metadata from remote
 	o.hasMetaData = false
-	return o.readMetaData(ctx)
+	err = o.readMetaData(ctx)
+	if err != nil {
+		return err
+	}
+	// The server only stores a checksum supplied with the upload,
+	// so ask the server to calculate one if it is missing.
+	if o.fs.canRecalcHash && o.sha1 == "" && o.md5 == "" {
+		err = o.recalculateHash(ctx)
+		if err != nil {
+			return fmt.Errorf("couldn't calculate checksum after upload: %w", err)
+		}
+	}
+	return nil
+}
+
+// recalculateHash asks the server to calculate and store the checksum
+// of the object's contents, updating the cached hash from the result.
+//
+// This uses the nextcloud PATCH extension with the X-Recalculate-Hash
+// header which returns the checksum in the OC-Checksum header.
+func (o *Object) recalculateHash(ctx context.Context) error {
+	hashName := ""
+	if o.fs.hasOCSHA1 {
+		hashName = "sha1"
+	} else if o.fs.hasOCMD5 {
+		hashName = "md5"
+	} else {
+		return nil
+	}
+	opts := rest.Opts{
+		Method:       "PATCH",
+		Path:         o.filePath(),
+		NoResponse:   true,
+		ExtraHeaders: map[string]string{"X-Recalculate-Hash": hashName},
+	}
+	var resp *http.Response
+	err := o.fs.pacer.Call(func() (bool, error) {
+		var err error
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		return o.fs.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return err
+	}
+	hashes := (&api.Prop{Checksums: []string{resp.Header.Get("OC-Checksum")}}).Hashes()
+	o.sha1 = hashes[hash.SHA1]
+	o.md5 = hashes[hash.MD5]
+	return nil
 }
 
 func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string]string {

docs/content/webdav.md

@@ -119,7 +119,9 @@ Likewise plain WebDAV does not support hashes, however when used with
 Fastmail Files, ownCloud or Nextcloud rclone will support SHA1 and MD5 hashes.
 Depending on the exact version of ownCloud or Nextcloud hashes may
 appear on all objects, or only on objects which had a hash uploaded
-with them.
+with them. With Nextcloud, rclone asks the server to calculate the SHA1
+of uploads which had no hash to send, such as streamed uploads, and
+after setting the modification time, which discards the stored hash.
 
 <!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/webdav/webdav.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
 ### Standard options