f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add context to all http.NewRequest #3257

Browse Source 
When we drop support for go1.12 we can use http.NewRequestWithContext
This commit is contained in:
Nick Craig-Wood
2019-09-09 23:27:07 +01:00
parent 58a531a203
commit 66c23723e3
13 changed files with 70 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/drive/drive.go
+12 -11
View File
@@ -1734,7 +1734,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
}
} else {
// Upload the file in chunks
info, err = f.Upload(in, size, srcMimeType, "", remote, createInfo)
info, err = f.Upload(ctx, in, size, srcMimeType, "", remote, createInfo)
if err != nil {
return nil, err
}
@@ -2499,7 +2499,7 @@ func (o *baseObject) Storable() bool {
// httpResponse gets an http.Response object for the object
// using the url and method passed in
func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
if url == "" {
return nil, nil, errors.New("forbidden to download - check sharing permission")
}
@@ -2507,6 +2507,7 @@ func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (
if err != nil {
return req, nil, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
fs.OpenOptionAddHTTPHeaders(req.Header, options)
if o.bytes == 0 {
// Don't supply range requests for 0 length objects as they always fail
@@ -2577,8 +2578,8 @@ func isGoogleError(err error, what string) bool {
}
// open a url for reading
func (o *baseObject) open(url string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
_, res, err := o.httpResponse(url, "GET", options)
func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
_, res, err := o.httpResponse(ctx, url, "GET", options)
if err != nil {
if isGoogleError(err, "cannotDownloadAbusiveFile") {
if o.fs.opt.AcknowledgeAbuse {
@@ -2589,7 +2590,7 @@ func (o *baseObject) open(url string, options ...fs.OpenOption) (in io.ReadClose
url += "?"
}
url += "acknowledgeAbuse=true"
_, res, err = o.httpResponse(url, "GET", options)
_, res, err = o.httpResponse(ctx, url, "GET", options)
} else {
err = errors.Wrap(err, "Use the --drive-acknowledge-abuse flag to download this file")
}
@@ -2618,7 +2619,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
o.v2Download = false
}
}
return o.baseObject.open(o.url, options...)
return o.baseObject.open(ctx, o.url, options...)
}
func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
// Update the size with what we are reading as it can change from
@@ -2643,7 +2644,7 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in
if offset != 0 {
return nil, errors.New("partial downloads are not supported while exporting Google Documents")
}
in, err = o.baseObject.open(o.url, options...)
in, err = o.baseObject.open(ctx, o.url, options...)
if in != nil {
in = &openDocumentFile{o: o, in: in}
}
@@ -2678,7 +2679,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
return ioutil.NopCloser(bytes.NewReader(data)), nil
}
func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io.Reader,
func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
src fs.ObjectInfo) (info *drive.File, err error) {
// Make the API request to upload metadata and file data.
size := src.Size()
@@ -2696,7 +2697,7 @@ func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io
return
}
// Upload the file in chunks
return o.fs.Upload(in, size, uploadMimeType, o.id, o.remote, updateInfo)
return o.fs.Upload(ctx, in, size, uploadMimeType, o.id, o.remote, updateInfo)
}
// Update the already existing object
@@ -2710,7 +2711,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
MimeType: srcMimeType,
ModifiedTime: src.ModTime(ctx).Format(timeFormatOut),
}
info, err := o.baseObject.update(updateInfo, srcMimeType, in, src)
info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
if err != nil {
return err
}
@@ -2747,7 +2748,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
}
updateInfo.MimeType = importMimeType
info, err := o.baseObject.update(updateInfo, srcMimeType, in, src)
info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
if err != nil {
return err
}
backend/drive/upload.go
+12 -9
View File
@@ -11,6 +11,7 @@
package drive
import (
"context"
"encoding/json"
"fmt"
"io"
@@ -50,7 +51,7 @@ type resumableUpload struct {
}
// Upload the io.Reader in of size bytes with contentType and info
func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
params := url.Values{
"alt": {"json"},
"uploadType": {"resumable"},
@@ -81,6 +82,7 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string
if err != nil {
return false, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
googleapi.Expand(req.URL, map[string]string{
"fileId": fileID,
})
@@ -106,12 +108,13 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string
MediaType: contentType,
ContentLength: size,
}
return rx.Upload()
return rx.Upload(ctx)
}
// Make an http.Request for the range passed in
func (rx *resumableUpload) makeRequest(start int64, body io.ReadSeeker, reqSize int64) *http.Request {
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
req, _ := http.NewRequest("POST", rx.URI, body)
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
req.ContentLength = reqSize
if reqSize != 0 {
req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
@@ -129,8 +132,8 @@ var rangeRE = regexp.MustCompile(`^0\-(\d+)$`)
// Query drive for the amount transferred so far
//
// If error is nil, then start should be valid
func (rx *resumableUpload) transferStatus() (start int64, err error) {
req := rx.makeRequest(0, nil, 0)
func (rx *resumableUpload) transferStatus(ctx context.Context) (start int64, err error) {
req := rx.makeRequest(ctx, 0, nil, 0)
res, err := rx.f.client.Do(req)
if err != nil {
return 0, err
@@ -157,9 +160,9 @@ func (rx *resumableUpload) transferStatus() (start int64, err error) {
}
// Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
_, _ = chunk.Seek(0, io.SeekStart)
req := rx.makeRequest(start, chunk, chunkSize)
req := rx.makeRequest(ctx, start, chunk, chunkSize)
res, err := rx.f.client.Do(req)
if err != nil {
return 599, err
@@ -192,7 +195,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
// Upload uploads the chunks from the input
// It retries each chunk using the pacer and --low-level-retries
func (rx *resumableUpload) Upload() (*drive.File, error) {
func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
start := int64(0)
var StatusCode int
var err error
@@ -207,7 +210,7 @@ func (rx *resumableUpload) Upload() (*drive.File, error) {
// Transfer the chunk
err = rx.f.pacer.Call(func() (bool, error) {
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
StatusCode, err = rx.transferChunk(start, chunk, reqSize)
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
again, err := shouldRetry(err)
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
again = false