From 1e426b93a1e43ea66278c28416f228cbedbe9809 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 26 Nov 2019 18:07:01 +0100 Subject: [PATCH] update vendor/ --- .../github.com/cavaliercoder/grab/.travis.yml | 13 + vendor/github.com/cavaliercoder/grab/LICENSE | 26 + vendor/github.com/cavaliercoder/grab/Makefile | 29 + .../github.com/cavaliercoder/grab/README.md | 127 +++++ .../github.com/cavaliercoder/grab/client.go | 506 ++++++++++++++++++ vendor/github.com/cavaliercoder/grab/doc.go | 63 +++ vendor/github.com/cavaliercoder/grab/error.go | 42 ++ vendor/github.com/cavaliercoder/grab/grab.go | 64 +++ .../cavaliercoder/grab/rate_limiter.go | 12 + .../github.com/cavaliercoder/grab/request.go | 172 ++++++ .../github.com/cavaliercoder/grab/response.go | 224 ++++++++ .../github.com/cavaliercoder/grab/states.wsd | 111 ++++ .../github.com/cavaliercoder/grab/transfer.go | 81 +++ vendor/github.com/cavaliercoder/grab/util.go | 89 +++ vendor/modules.txt | 2 + 15 files changed, 1561 insertions(+) create mode 100644 vendor/github.com/cavaliercoder/grab/.travis.yml create mode 100644 vendor/github.com/cavaliercoder/grab/LICENSE create mode 100644 vendor/github.com/cavaliercoder/grab/Makefile create mode 100644 vendor/github.com/cavaliercoder/grab/README.md create mode 100644 vendor/github.com/cavaliercoder/grab/client.go create mode 100644 vendor/github.com/cavaliercoder/grab/doc.go create mode 100644 vendor/github.com/cavaliercoder/grab/error.go create mode 100644 vendor/github.com/cavaliercoder/grab/grab.go create mode 100644 vendor/github.com/cavaliercoder/grab/rate_limiter.go create mode 100644 vendor/github.com/cavaliercoder/grab/request.go create mode 100644 vendor/github.com/cavaliercoder/grab/response.go create mode 100644 vendor/github.com/cavaliercoder/grab/states.wsd create mode 100644 vendor/github.com/cavaliercoder/grab/transfer.go create mode 100644 vendor/github.com/cavaliercoder/grab/util.go diff --git a/vendor/github.com/cavaliercoder/grab/.travis.yml b/vendor/github.com/cavaliercoder/grab/.travis.yml new file mode 100644 index 00000000..efcdb147 --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/.travis.yml @@ -0,0 +1,13 @@ +language: go + +go: + - 1.10.x + - 1.9.x + - 1.8.x + - 1.7.x + +script: make check + +env: + - GOARCH=amd64 + - GOARCH=386 diff --git a/vendor/github.com/cavaliercoder/grab/LICENSE b/vendor/github.com/cavaliercoder/grab/LICENSE new file mode 100644 index 00000000..7f377a1b --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2017 Ryan Armstrong. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/cavaliercoder/grab/Makefile b/vendor/github.com/cavaliercoder/grab/Makefile new file mode 100644 index 00000000..8f6f289e --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/Makefile @@ -0,0 +1,29 @@ +GO = go +GOGET = $(GO) get -u + +all: check lint + +check: + cd cmd/grab && $(MAKE) -B all + $(GO) test -cover -race ./... + +install: + $(GO) install -v ./... + +clean: + $(GO) clean -x ./... + rm -rvf ./.test* + +lint: + gofmt -l -e -s . || : + go vet . || : + golint . || : + gocyclo -over 15 . || : + misspell ./* || : + +deps: + $(GOGET) github.com/golang/lint/golint + $(GOGET) github.com/fzipp/gocyclo + $(GOGET) github.com/client9/misspell/cmd/misspell + +.PHONY: all check install clean lint deps diff --git a/vendor/github.com/cavaliercoder/grab/README.md b/vendor/github.com/cavaliercoder/grab/README.md new file mode 100644 index 00000000..5066789f --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/README.md @@ -0,0 +1,127 @@ +# grab + +[![GoDoc](https://godoc.org/github.com/cavaliercoder/grab?status.svg)](https://godoc.org/github.com/cavaliercoder/grab) [![Build Status](https://travis-ci.org/cavaliercoder/grab.svg?branch=master)](https://travis-ci.org/cavaliercoder/grab) [![Go Report Card](https://goreportcard.com/badge/github.com/cavaliercoder/grab)](https://goreportcard.com/report/github.com/cavaliercoder/grab) + +*Downloading the internet, one goroutine at a time!* + + $ go get github.com/cavaliercoder/grab + +Grab is a Go package for downloading files from the internet with the following +rad features: + +* Monitor download progress concurrently +* Auto-resume incomplete downloads +* Guess filename from content header or URL path +* Safely cancel downloads using context.Context +* Validate downloads using checksums +* Download batches of files concurrently +* Apply rate limiters + +Requires Go v1.7+ + +## Example + +The following example downloads a PDF copy of the free eBook, "An Introduction +to Programming in Go" into the current working directory. + +```go +resp, err := grab.Get(".", "http://www.golang-book.com/public/pdf/gobook.pdf") +if err != nil { + log.Fatal(err) +} + +fmt.Println("Download saved to", resp.Filename) +``` + +The following, more complete example allows for more granular control and +periodically prints the download progress until it is complete. + +The second time you run the example, it will auto-resume the previous download +and exit sooner. + +```go +package main + +import ( + "fmt" + "os" + "time" + + "github.com/cavaliercoder/grab" +) + +func main() { + // create client + client := grab.NewClient() + req, _ := grab.NewRequest(".", "http://www.golang-book.com/public/pdf/gobook.pdf") + + // start download + fmt.Printf("Downloading %v...\n", req.URL()) + resp := client.Do(req) + fmt.Printf(" %v\n", resp.HTTPResponse.Status) + + // start UI loop + t := time.NewTicker(500 * time.Millisecond) + defer t.Stop() + +Loop: + for { + select { + case <-t.C: + fmt.Printf(" transferred %v / %v bytes (%.2f%%)\n", + resp.BytesComplete(), + resp.Size, + 100*resp.Progress()) + + case <-resp.Done: + // download is complete + break Loop + } + } + + // check for errors + if err := resp.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Download failed: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Download saved to ./%v \n", resp.Filename) + + // Output: + // Downloading http://www.golang-book.com/public/pdf/gobook.pdf... + // 200 OK + // transferred 42970 / 2893557 bytes (1.49%) + // transferred 1207474 / 2893557 bytes (41.73%) + // transferred 2758210 / 2893557 bytes (95.32%) + // Download saved to ./gobook.pdf +} +``` + +## Design trade-offs + +The primary use case for Grab is to concurrently downloading thousands of large +files from remote file repositories where the remote files are immutable. +Examples include operating system package repositories or ISO libraries. + +Grab aims to provide robust, sane defaults. These are usually determined using +the HTTP specifications, or by mimicking the behavior of common web clients like +cURL, wget and common web browsers. + +Grab aims to be stateless. The only state that exists is the remote files you +wish to download and the local copy which may be completed, partially completed +or not yet created. The advantage to this is that the local file system is not +cluttered unnecessarily with addition state files (like a `.crdownload` file). +The disadvantage of this approach is that grab must make assumptions about the +local and remote state; specifically, that they have not been modified by +another program. + +If the local or remote file are modified outside of grab, and you download the +file again with resuming enabled, the local file will likely become corrupted. +In this case, you might consider making remote files immutable, or disabling +resume. + +Grab aims to enable best-in-class functionality for more complex features +through extensible interfaces, rather than reimplementation. For example, +you can provide your own Hash algorithm to compute file checksums, or your +own rate limiter implementation (with all the associated trade-offs) to rate +limit downloads. diff --git a/vendor/github.com/cavaliercoder/grab/client.go b/vendor/github.com/cavaliercoder/grab/client.go new file mode 100644 index 00000000..62f4257d --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/client.go @@ -0,0 +1,506 @@ +package grab + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "sync" + "time" +) + +// A Client is a file download client. +// +// Clients are safe for concurrent use by multiple goroutines. +type Client struct { + // HTTPClient specifies the http.Client which will be used for communicating + // with the remote server during the file transfer. + HTTPClient *http.Client + + // UserAgent specifies the User-Agent string which will be set in the + // headers of all requests made by this client. + // + // The user agent string may be overridden in the headers of each request. + UserAgent string + + // BufferSize specifies the size in bytes of the buffer that is used for + // transferring all requested files. Larger buffers may result in faster + // throughput but will use more memory and result in less frequent updates + // to the transfer progress statistics. The BufferSize of each request can + // be overridden on each Request object. Default: 32KB. + BufferSize int +} + +// NewClient returns a new file download Client, using default configuration. +func NewClient() *Client { + return &Client{ + UserAgent: "grab", + HTTPClient: &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + }, + }, + } +} + +// DefaultClient is the default client and is used by all Get convenience +// functions. +var DefaultClient = NewClient() + +// Do sends a file transfer request and returns a file transfer response, +// following policy (e.g. redirects, cookies, auth) as configured on the +// client's HTTPClient. +// +// Like http.Get, Do blocks while the transfer is initiated, but returns as soon +// as the transfer has started transferring in a background goroutine, or if it +// failed early. +// +// An error is returned via Response.Err if caused by client policy (such as +// CheckRedirect), or if there was an HTTP protocol or IO error. Response.Err +// will block the caller until the transfer is completed, successfully or +// otherwise. +func (c *Client) Do(req *Request) *Response { + // cancel will be called on all code-paths via closeResponse + ctx, cancel := context.WithCancel(req.Context()) + resp := &Response{ + Request: req, + Start: time.Now(), + Done: make(chan struct{}, 0), + Filename: req.Filename, + ctx: ctx, + cancel: cancel, + bufferSize: req.BufferSize, + } + if resp.bufferSize == 0 { + // default to Client.BufferSize + resp.bufferSize = c.BufferSize + } + + // Run state-machine while caller is blocked to initialize the file transfer. + // Must never transition to the copyFile state - this happens next in another + // goroutine. + c.run(resp, c.statFileInfo) + + // Run copyFile in a new goroutine. copyFile will no-op if the transfer is + // already complete or failed. + go c.run(resp, c.copyFile) + return resp +} + +// DoChannel executes all requests sent through the given Request channel, one +// at a time, until it is closed by another goroutine. The caller is blocked +// until the Request channel is closed and all transfers have completed. All +// responses are sent through the given Response channel as soon as they are +// received from the remote servers and can be used to track the progress of +// each download. +// +// Slow Response receivers will cause a worker to block and therefore delay the +// start of the transfer for an already initiated connection - potentially +// causing a server timeout. It is the caller's responsibility to ensure a +// sufficient buffer size is used for the Response channel to prevent this. +// +// If an error occurs during any of the file transfers it will be accessible via +// the associated Response.Err function. +func (c *Client) DoChannel(reqch <-chan *Request, respch chan<- *Response) { + // TODO: enable cancelling of batch jobs + for req := range reqch { + resp := c.Do(req) + respch <- resp + <-resp.Done + } +} + +// DoBatch executes all the given requests using the given number of concurrent +// workers. Control is passed back to the caller as soon as the workers are +// initiated. +// +// If the requested number of workers is less than one, a worker will be created +// for every request. I.e. all requests will be executed concurrently. +// +// If an error occurs during any of the file transfers it will be accessible via +// call to the associated Response.Err. +// +// The returned Response channel is closed only after all of the given Requests +// have completed, successfully or otherwise. +func (c *Client) DoBatch(workers int, requests ...*Request) <-chan *Response { + if workers < 1 { + workers = len(requests) + } + reqch := make(chan *Request, len(requests)) + respch := make(chan *Response, len(requests)) + wg := sync.WaitGroup{} + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + c.DoChannel(reqch, respch) + wg.Done() + }() + } + + // queue requests + go func() { + for _, req := range requests { + reqch <- req + } + close(reqch) + wg.Wait() + close(respch) + }() + return respch +} + +// An stateFunc is an action that mutates the state of a Response and returns +// the next stateFunc to be called. +type stateFunc func(*Response) stateFunc + +// run calls the given stateFunc function and all subsequent returned stateFuncs +// until a stateFunc returns nil or the Response.ctx is canceled. Each stateFunc +// should mutate the state of the given Response until it has completed +// downloading or failed. +func (c *Client) run(resp *Response, f stateFunc) { + for { + select { + case <-resp.ctx.Done(): + if resp.IsComplete() { + return + } + resp.err = resp.ctx.Err() + f = c.closeResponse + + default: + // keep working + } + if f = f(resp); f == nil { + return + } + } +} + +// statFileInfo retrieves FileInfo for any local file matching +// Response.Filename. +// +// If the file does not exist, is a directory, or its name is unknown the next +// stateFunc is headRequest. +// +// If the file exists, Response.fi is set and the next stateFunc is +// validateLocal. +// +// If an error occurs, the next stateFunc is closeResponse. +func (c *Client) statFileInfo(resp *Response) stateFunc { + if resp.Filename == "" { + return c.headRequest + } + fi, err := os.Stat(resp.Filename) + if err != nil { + if os.IsNotExist(err) { + return c.headRequest + } + resp.err = err + return c.closeResponse + } + if fi.IsDir() { + resp.Filename = "" + return c.headRequest + } + resp.fi = fi + return c.validateLocal +} + +// validateLocal compares a local copy of the downloaded file to the remote +// file. +// +// An error is returned if the local file is larger than the remote file, or +// Request.SkipExisting is true. +// +// If the existing file matches the length of the remote file, the next +// stateFunc is checksumFile. +// +// If the local file is smaller than the remote file and the remote server is +// known to support ranged requests, the next stateFunc is getRequest. +func (c *Client) validateLocal(resp *Response) stateFunc { + if resp.Request.SkipExisting { + resp.err = ErrFileExists + return c.closeResponse + } + + // determine expected file size + size := resp.Request.Size + if size == 0 && resp.HTTPResponse != nil { + size = resp.HTTPResponse.ContentLength + } + if size == 0 { + return c.headRequest + } + + if size == resp.fi.Size() { + resp.DidResume = true + resp.bytesResumed = resp.fi.Size() + return c.checksumFile + } + + if resp.Request.NoResume { + return c.getRequest + } + + if size < resp.fi.Size() { + resp.err = ErrBadLength + return c.closeResponse + } + + if resp.CanResume { + resp.Request.HTTPRequest.Header.Set( + "Range", + fmt.Sprintf("bytes=%d-", resp.fi.Size())) + resp.DidResume = true + resp.bytesResumed = resp.fi.Size() + return c.getRequest + } + return c.headRequest +} + +func (c *Client) checksumFile(resp *Response) stateFunc { + if resp.Request.hash == nil { + return c.closeResponse + } + if resp.Filename == "" { + panic("filename not set") + } + req := resp.Request + + // compare checksum + var sum []byte + sum, resp.err = checksum(req.Context(), resp.Filename, req.hash) + if resp.err != nil { + return c.closeResponse + } + if !bytes.Equal(sum, req.checksum) { + resp.err = ErrBadChecksum + if req.deleteOnError { + if err := os.Remove(resp.Filename); err != nil { + // err should be os.PathError and include file path + resp.err = fmt.Errorf( + "cannot remove downloaded file with checksum mismatch: %v", + err) + } + } + } + return c.closeResponse +} + +// doHTTPRequest sends a HTTP Request and returns the response +func (c *Client) doHTTPRequest(req *http.Request) (*http.Response, error) { + if c.UserAgent != "" && req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", c.UserAgent) + } + return c.HTTPClient.Do(req) +} + +func (c *Client) headRequest(resp *Response) stateFunc { + if resp.optionsKnown { + return c.getRequest + } + resp.optionsKnown = true + + if resp.Request.NoResume { + return c.getRequest + } + + if resp.Filename != "" && resp.fi == nil { + // destination path is already known and does not exist + return c.getRequest + } + + hreq := new(http.Request) + *hreq = *resp.Request.HTTPRequest + hreq.Method = "HEAD" + + resp.HTTPResponse, resp.err = c.doHTTPRequest(hreq) + if resp.err != nil { + return c.closeResponse + } + resp.HTTPResponse.Body.Close() + + if resp.HTTPResponse.StatusCode != http.StatusOK { + return c.getRequest + } + + return c.readResponse +} + +func (c *Client) getRequest(resp *Response) stateFunc { + resp.HTTPResponse, resp.err = c.doHTTPRequest(resp.Request.HTTPRequest) + if resp.err != nil { + return c.closeResponse + } + + // check status code + if !resp.Request.IgnoreBadStatusCodes { + if resp.HTTPResponse.StatusCode < 200 || resp.HTTPResponse.StatusCode > 299 { + resp.err = StatusCodeError(resp.HTTPResponse.StatusCode) + return c.closeResponse + } + } + + return c.readResponse +} + +func (c *Client) readResponse(resp *Response) stateFunc { + if resp.HTTPResponse == nil { + panic("Response.HTTPResponse is not ready") + } + + // check expected size + resp.Size = resp.bytesResumed + resp.HTTPResponse.ContentLength + if resp.HTTPResponse.ContentLength > 0 && resp.Request.Size > 0 { + if resp.Request.Size != resp.Size { + resp.err = ErrBadLength + return c.closeResponse + } + } + + // check filename + if resp.Filename == "" { + filename, err := guessFilename(resp.HTTPResponse) + if err != nil { + resp.err = err + return c.closeResponse + } + // Request.Filename will be empty or a directory + resp.Filename = filepath.Join(resp.Request.Filename, filename) + } + + if resp.requestMethod() == "HEAD" { + if resp.HTTPResponse.Header.Get("Accept-Ranges") == "bytes" { + resp.CanResume = true + } + return c.statFileInfo + } + return c.openWriter +} + +// openWriter opens the destination file for writing and seeks to the location +// from whence the file transfer will resume. +// +// Requires that Response.Filename and resp.DidResume are already be set. +func (c *Client) openWriter(resp *Response) stateFunc { + if !resp.Request.NoCreateDirectories { + resp.err = mkdirp(resp.Filename) + if resp.err != nil { + return c.closeResponse + } + } + + // compute write flags + flag := os.O_CREATE | os.O_WRONLY + if resp.fi != nil { + if resp.DidResume { + flag = os.O_APPEND | os.O_WRONLY + } else { + flag = os.O_TRUNC | os.O_WRONLY + } + } + + // open file + f, err := os.OpenFile(resp.Filename, flag, 0644) + if err != nil { + resp.err = err + return c.closeResponse + } + resp.writer = f + + // seek to start or end + whence := os.SEEK_SET + if resp.bytesResumed > 0 { + whence = os.SEEK_END + } + _, resp.err = f.Seek(0, whence) + if resp.err != nil { + return c.closeResponse + } + + // init transfer + if resp.bufferSize < 1 { + resp.bufferSize = 32 * 1024 + } + b := make([]byte, resp.bufferSize) + resp.transfer = newTransfer( + resp.Request.Context(), + resp.Request.RateLimiter, + resp.writer, + resp.HTTPResponse.Body, + b) + + // next step is copyFile, but this will be called later in another goroutine + return nil +} + +// copy transfers content for a HTTP connection established via Client.do() +func (c *Client) copyFile(resp *Response) stateFunc { + if resp.IsComplete() { + return nil + } + + // run BeforeCopy hook + if f := resp.Request.BeforeCopy; f != nil { + resp.err = f(resp) + if resp.err != nil { + return c.closeResponse + } + } + + if resp.transfer == nil { + panic("developer error: Response.transfer is not initialized") + } + go resp.watchBps() + _, resp.err = resp.transfer.copy() + if resp.err != nil { + return c.closeResponse + } + closeWriter(resp) + + // set timestamp + if !resp.Request.IgnoreRemoteTime { + resp.err = setLastModified(resp.HTTPResponse, resp.Filename) + if resp.err != nil { + return c.closeResponse + } + } + + // run AfterCopy hook + if f := resp.Request.AfterCopy; f != nil { + resp.err = f(resp) + if resp.err != nil { + return c.closeResponse + } + } + + return c.checksumFile +} + +func closeWriter(resp *Response) { + if resp.writer != nil { + resp.writer.Close() + resp.writer = nil + } +} + +// close finalizes the Response +func (c *Client) closeResponse(resp *Response) stateFunc { + if resp.IsComplete() { + panic("response already closed") + } + + resp.fi = nil + closeWriter(resp) + resp.closeResponseBody() + + resp.End = time.Now() + close(resp.Done) + if resp.cancel != nil { + resp.cancel() + } + + return nil +} diff --git a/vendor/github.com/cavaliercoder/grab/doc.go b/vendor/github.com/cavaliercoder/grab/doc.go new file mode 100644 index 00000000..4892d079 --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/doc.go @@ -0,0 +1,63 @@ +/* +Package grab provides a HTTP download manager implementation. + +Get is the most simple way to download a file: + + resp, err := grab.Get("/tmp", "http://example.com/example.zip") + // ... + +Get will download the given URL and save it to the given destination directory. +The destination filename will be determined automatically by grab using +Content-Disposition headers returned by the remote server, or by inspecting the +requested URL path. + +An empty destination string or "." means the transfer will be stored in the +current working directory. + +If a destination file already exists, grab will assume it is a complete or +partially complete download of the requested file. If the remote server supports +resuming interrupted downloads, grab will resume downloading from the end of the +partial file. If the server does not support resumed downloads, the file will be +retransferred in its entirety. If the file is already complete, grab will return +successfully. + +For control over the HTTP client, destination path, auto-resume, checksum +validation and other settings, create a Client: + + client := grab.NewClient() + client.HTTPClient.Transport.DisableCompression = true + + req, err := grab.NewRequest("/tmp", "http://example.com/example.zip") + // ... + req.NoResume = true + req.HTTPRequest.Header.Set("Authorization", "Basic YWxhZGRpbjpvcGVuc2VzYW1l") + + resp := client.Do(req) + // ... + +You can monitor the progress of downloads while they are transferring: + + client := grab.NewClient() + req, err := grab.NewRequest("", "http://example.com/example.zip") + // ... + resp := client.Do(req) + + t := time.NewTicker(time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + fmt.Printf("%.02f%% complete\n", resp.Progress()) + + case <-resp.Done: + if err := resp.Err(); err != nil { + // ... + } + + // ... + return + } + } +*/ +package grab diff --git a/vendor/github.com/cavaliercoder/grab/error.go b/vendor/github.com/cavaliercoder/grab/error.go new file mode 100644 index 00000000..db76be7d --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/error.go @@ -0,0 +1,42 @@ +package grab + +import ( + "errors" + "fmt" + "net/http" +) + +var ( + // ErrBadLength indicates that the server response or an existing file does + // not match the expected content length. + ErrBadLength = errors.New("bad content length") + + // ErrBadChecksum indicates that a downloaded file failed to pass checksum + // validation. + ErrBadChecksum = errors.New("checksum mismatch") + + // ErrNoFilename indicates that a reasonable filename could not be + // automatically determined using the URL or response headers from a server. + ErrNoFilename = errors.New("no filename could be determined") + + // ErrNoTimestamp indicates that a timestamp could not be automatically + // determined using the response headers from the remote server. + ErrNoTimestamp = errors.New("no timestamp could be determined for the remote file") + + // ErrFileExists indicates that the destination path already exists. + ErrFileExists = errors.New("file exists") +) + +// StatusCodeError indicates that the server response had a status code that +// was not in the 200-299 range (after following any redirects). +type StatusCodeError int + +func (err StatusCodeError) Error() string { + return fmt.Sprintf("server returned %d %s", err, http.StatusText(int(err))) +} + +// IsStatusCodeError returns true if the given error is of type StatusCodeError. +func IsStatusCodeError(err error) bool { + _, ok := err.(StatusCodeError) + return ok +} diff --git a/vendor/github.com/cavaliercoder/grab/grab.go b/vendor/github.com/cavaliercoder/grab/grab.go new file mode 100644 index 00000000..edd27880 --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/grab.go @@ -0,0 +1,64 @@ +package grab + +import ( + "fmt" + "os" +) + +// Get sends a HTTP request and downloads the content of the requested URL to +// the given destination file path. The caller is blocked until the download is +// completed, successfully or otherwise. +// +// An error is returned if caused by client policy (such as CheckRedirect), or +// if there was an HTTP protocol or IO error. +// +// For non-blocking calls or control over HTTP client headers, redirect policy, +// and other settings, create a Client instead. +func Get(dst, urlStr string) (*Response, error) { + req, err := NewRequest(dst, urlStr) + if err != nil { + return nil, err + } + + resp := DefaultClient.Do(req) + return resp, resp.Err() +} + +// GetBatch sends multiple HTTP requests and downloads the content of the +// requested URLs to the given destination directory using the given number of +// concurrent worker goroutines. +// +// The Response for each requested URL is sent through the returned Response +// channel, as soon as a worker receives a response from the remote server. The +// Response can then be used to track the progress of the download while it is +// in progress. +// +// The returned Response channel will be closed by Grab, only once all downloads +// have completed or failed. +// +// If an error occurs during any download, it will be available via call to the +// associated Response.Err. +// +// For control over HTTP client headers, redirect policy, and other settings, +// create a Client instead. +func GetBatch(workers int, dst string, urlStrs ...string) (<-chan *Response, error) { + fi, err := os.Stat(dst) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("destination is not a directory") + } + + reqs := make([]*Request, len(urlStrs)) + for i := 0; i < len(urlStrs); i++ { + req, err := NewRequest(dst, urlStrs[i]) + if err != nil { + return nil, err + } + reqs[i] = req + } + + ch := DefaultClient.DoBatch(workers, reqs...) + return ch, nil +} diff --git a/vendor/github.com/cavaliercoder/grab/rate_limiter.go b/vendor/github.com/cavaliercoder/grab/rate_limiter.go new file mode 100644 index 00000000..6c1fb31b --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/rate_limiter.go @@ -0,0 +1,12 @@ +package grab + +import "context" + +// RateLimiter is an interface that must be satisfied by any third-party rate +// limiters that may be used to limit download transfer speeds. +// +// A recommended token bucket implementation can be found at +// https://godoc.org/golang.org/x/time/rate#Limiter. +type RateLimiter interface { + WaitN(ctx context.Context, n int) (err error) +} diff --git a/vendor/github.com/cavaliercoder/grab/request.go b/vendor/github.com/cavaliercoder/grab/request.go new file mode 100644 index 00000000..50745510 --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/request.go @@ -0,0 +1,172 @@ +package grab + +import ( + "context" + "hash" + "net/http" + "net/url" +) + +// A Hook is a user provided callback function that can be called by grab at +// various stages of a requests lifecycle. If a hook returns an error, the +// associated request is canceled and the same error is returned on the Response +// object. +// +// Hook functions are called synchronously and should never block unnecessarily. +// Response methods that block until a download is complete, such as +// Response.Err, Response.Cancel or Response.Wait will deadlock. To cancel a +// download from a callback, simply return a non-nil error. +type Hook func(*Response) error + +// A Request represents an HTTP file transfer request to be sent by a Client. +type Request struct { + // Label is an arbitrary string which may used to label a Request with a + // user friendly name. + Label string + + // Tag is an arbitrary interface which may be used to relate a Request to + // other data. + Tag interface{} + + // HTTPRequest specifies the http.Request to be sent to the remote server to + // initiate a file transfer. It includes request configuration such as URL, + // protocol version, HTTP method, request headers and authentication. + HTTPRequest *http.Request + + // Filename specifies the path where the file transfer will be stored in + // local storage. If Filename is empty or a directory, the true Filename will + // be resolved using Content-Disposition headers or the request URL. + // + // An empty string means the transfer will be stored in the current working + // directory. + Filename string + + // SkipExisting specifies that ErrFileExists should be returned if the + // destination path already exists. The existing file will not be checked for + // completeness. + SkipExisting bool + + // NoResume specifies that a partially completed download will be restarted + // without attempting to resume any existing file. If the download is already + // completed in full, it will not be restarted. + NoResume bool + + // NoCreateDirectories specifies that any missing directories in the given + // Filename path should not be created automatically, if they do not already + // exist. + NoCreateDirectories bool + + // IgnoreBadStatusCodes specifies that grab should accept any status code in + // the response from the remote server. Otherwise, grab expects the response + // status code to be within the 2XX range (after following redirects). + IgnoreBadStatusCodes bool + + // IgnoreRemoteTime specifies that grab should not attempt to set the + // timestamp of the local file to match the remote file. + IgnoreRemoteTime bool + + // Size specifies the expected size of the file transfer if known. If the + // server response size does not match, the transfer is cancelled and + // ErrBadLength returned. + Size int64 + + // BufferSize specifies the size in bytes of the buffer that is used for + // transferring the requested file. Larger buffers may result in faster + // throughput but will use more memory and result in less frequent updates + // to the transfer progress statistics. If a RateLimiter is configured, + // BufferSize should be much lower than the rate limit. Default: 32KB. + BufferSize int + + // RateLimiter allows the transfer rate of a download to be limited. The given + // Request.BufferSize determines how frequently the RateLimiter will be + // polled. + RateLimiter RateLimiter + + // BeforeCopy is a user provided callback that is called immediately before + // a request starts downloading. If BeforeCopy returns an error, the request + // is cancelled and the same error is returned on the Response object. + BeforeCopy Hook + + // AfterCopy is a user provided callback that is called immediately after a + // request has finished downloading, before checksum validation and closure. + // This hook is only called if the transfer was successful. If AfterCopy + // returns an error, the request is canceled and the same error is returned on + // the Response object. + AfterCopy Hook + + // hash, checksum and deleteOnError - set via SetChecksum. + hash hash.Hash + checksum []byte + deleteOnError bool + + // Context for cancellation and timeout - set via WithContext + ctx context.Context +} + +// NewRequest returns a new file transfer Request suitable for use with +// Client.Do. +func NewRequest(dst, urlStr string) (*Request, error) { + if dst == "" { + dst = "." + } + req, err := http.NewRequest("GET", urlStr, nil) + if err != nil { + return nil, err + } + return &Request{ + HTTPRequest: req, + Filename: dst, + }, nil +} + +// Context returns the request's context. To change the context, use +// WithContext. +// +// The returned context is always non-nil; it defaults to the background +// context. +// +// The context controls cancelation. +func (r *Request) Context() context.Context { + if r.ctx != nil { + return r.ctx + } + + return context.Background() +} + +// WithContext returns a shallow copy of r with its context changed +// to ctx. The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") + } + r2 := new(Request) + *r2 = *r + r2.ctx = ctx + r2.HTTPRequest = r2.HTTPRequest.WithContext(ctx) + return r2 +} + +// URL returns the URL to be downloaded. +func (r *Request) URL() *url.URL { + return r.HTTPRequest.URL +} + +// SetChecksum sets the desired hashing algorithm and checksum value to validate +// a downloaded file. Once the download is complete, the given hashing algorithm +// will be used to compute the actual checksum of the downloaded file. If the +// checksums do not match, an error will be returned by the associated +// Response.Err method. +// +// If deleteOnError is true, the downloaded file will be deleted automatically +// if it fails checksum validation. +// +// To prevent corruption of the computed checksum, the given hash must not be +// used by any other request or goroutines. +// +// To disable checksum validation, call SetChecksum with a nil hash. +func (r *Request) SetChecksum(h hash.Hash, sum []byte, deleteOnError bool) { + r.hash = h + r.checksum = sum + r.deleteOnError = deleteOnError +} diff --git a/vendor/github.com/cavaliercoder/grab/response.go b/vendor/github.com/cavaliercoder/grab/response.go new file mode 100644 index 00000000..9a30ee3b --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/response.go @@ -0,0 +1,224 @@ +package grab + +import ( + "context" + "io" + "net/http" + "os" + "sync" + "time" +) + +// Response represents the response to a completed or in-progress download +// request. +// +// A response may be returned as soon a HTTP response is received from a remote +// server, but before the body content has started transferring. +// +// All Response method calls are thread-safe. +type Response struct { + // The Request that was submitted to obtain this Response. + Request *Request + + // HTTPResponse represents the HTTP response received from an HTTP request. + // + // The response Body should not be used as it will be consumed and closed by + // grab. + HTTPResponse *http.Response + + // Filename specifies the path where the file transfer is stored in local + // storage. + Filename string + + // Size specifies the total expected size of the file transfer. + Size int64 + + // Start specifies the time at which the file transfer started. + Start time.Time + + // End specifies the time at which the file transfer completed. + // + // This will return zero until the transfer has completed. + End time.Time + + // CanResume specifies that the remote server advertised that it can resume + // previous downloads, as the 'Accept-Ranges: bytes' header is set. + CanResume bool + + // DidResume specifies that the file transfer resumed a previously incomplete + // transfer. + DidResume bool + + // Done is closed once the transfer is finalized, either successfully or with + // errors. Errors are available via Response.Err + Done chan struct{} + + // ctx is a Context that controls cancelation of an inprogress transfer + ctx context.Context + + // cancel is a cancel func that can be used to cancel the context of this + // Response. + cancel context.CancelFunc + + // fi is the FileInfo for the destination file if it already existed before + // transfer started. + fi os.FileInfo + + // optionsKnown indicates that a HEAD request has been completed and the + // capabilities of the remote server are known. + optionsKnown bool + + // writer is the file handle used to write the downloaded file to local + // storage + writer io.WriteCloser + + // bytesCompleted specifies the number of bytes which were already + // transferred before this transfer began. + bytesResumed int64 + + // transfer is responsible for copying data from the remote server to a local + // file, tracking progress and allowing for cancelation. + transfer *transfer + + // bytesPerSecond specifies the number of bytes that have been transferred in + // the last 1-second window. + bytesPerSecond float64 + bytesPerSecondMu sync.Mutex + + // bufferSize specifies the size in bytes of the transfer buffer. + bufferSize int + + // Error contains any error that may have occurred during the file transfer. + // This should not be read until IsComplete returns true. + err error +} + +// IsComplete returns true if the download has completed. If an error occurred +// during the download, it can be returned via Err. +func (c *Response) IsComplete() bool { + select { + case <-c.Done: + return true + default: + return false + } +} + +// Cancel cancels the file transfer by canceling the underlying Context for +// this Response. Cancel blocks until the transfer is closed and returns any +// error - typically context.Canceled. +func (c *Response) Cancel() error { + c.cancel() + return c.Err() +} + +// Wait blocks until the download is completed. +func (c *Response) Wait() { + <-c.Done +} + +// Err blocks the calling goroutine until the underlying file transfer is +// completed and returns any error that may have occurred. If the download is +// already completed, Err returns immediately. +func (c *Response) Err() error { + <-c.Done + return c.err +} + +// BytesComplete returns the total number of bytes which have been copied to +// the destination, including any bytes that were resumed from a previous +// download. +func (c *Response) BytesComplete() int64 { + return c.bytesResumed + c.transfer.N() +} + +// BytesPerSecond returns the number of bytes transferred in the last second. If +// the download is already complete, the average bytes/sec for the life of the +// download is returned. +func (c *Response) BytesPerSecond() float64 { + if c.IsComplete() { + return float64(c.transfer.N()) / c.Duration().Seconds() + } + c.bytesPerSecondMu.Lock() + defer c.bytesPerSecondMu.Unlock() + return c.bytesPerSecond +} + +// Progress returns the ratio of total bytes that have been downloaded. Multiply +// the returned value by 100 to return the percentage completed. +func (c *Response) Progress() float64 { + if c.Size == 0 { + return 0 + } + return float64(c.BytesComplete()) / float64(c.Size) +} + +// Duration returns the duration of a file transfer. If the transfer is in +// process, the duration will be between now and the start of the transfer. If +// the transfer is complete, the duration will be between the start and end of +// the completed transfer process. +func (c *Response) Duration() time.Duration { + if c.IsComplete() { + return c.End.Sub(c.Start) + } + + return time.Now().Sub(c.Start) +} + +// ETA returns the estimated time at which the the download will complete, given +// the current BytesPerSecond. If the transfer has already completed, the actual +// end time will be returned. +func (c *Response) ETA() time.Time { + if c.IsComplete() { + return c.End + } + bt := c.BytesComplete() + bps := c.BytesPerSecond() + if bps == 0 { + return time.Time{} + } + secs := float64(c.Size-bt) / bps + return time.Now().Add(time.Duration(secs) * time.Second) +} + +// watchBps watches the progress of a transfer and maintains statistics. +func (c *Response) watchBps() { + var prev int64 + then := c.Start + + t := time.NewTicker(time.Second) + defer t.Stop() + + for { + select { + case <-c.Done: + return + + case now := <-t.C: + d := now.Sub(then) + then = now + + cur := c.transfer.N() + bs := cur - prev + prev = cur + + c.bytesPerSecondMu.Lock() + c.bytesPerSecond = float64(bs) / d.Seconds() + c.bytesPerSecondMu.Unlock() + } + } +} + +func (c *Response) requestMethod() string { + if c == nil || c.HTTPResponse == nil || c.HTTPResponse.Request == nil { + return "" + } + return c.HTTPResponse.Request.Method +} + +func (c *Response) closeResponseBody() error { + if c.HTTPResponse == nil || c.HTTPResponse.Body == nil { + return nil + } + return c.HTTPResponse.Body.Close() +} diff --git a/vendor/github.com/cavaliercoder/grab/states.wsd b/vendor/github.com/cavaliercoder/grab/states.wsd new file mode 100644 index 00000000..9390a3f5 --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/states.wsd @@ -0,0 +1,111 @@ +@startuml +title Grab transfer state + +legend +| # | Meaning | +| D | Destination path known | +| S | File size known | +| O | Server options known (Accept-Ranges) | +| R | Resume supported (Accept-Ranges) | +| Z | Local file empty or missing | +| P | Local file partially complete | +endlegend + +[*] --> Empty +[*] --> D +[*] --> S +[*] --> DS + +Empty : Filename: "" +Empty : Size: 0 +Empty --> O : HEAD: Method not allowed +Empty --> DSO : HEAD: Range not supported +Empty --> DSOR : HEAD: Range supported + +DS : Filename: "foo.bar" +DS : Size: > 0 +DS --> DSZ : checkExisting(): File missing +DS --> DSP : checkExisting(): File partial +DS --> [*] : checkExisting(): File complete +DS --> ERROR + +S : Filename: "" +S : Size: > 0 +S --> SO : HEAD: Method not allowed +S --> DSO : HEAD: Range not supported +S --> DSOR : HEAD: Range supported + +D : Filename: "foo.bar" +D : Size: 0 +D --> DO : HEAD: Method not allowed +D --> DSO : HEAD: Range not supported +D --> DSOR : HEAD: Range supported + + +O : Filename: "" +O : Size: 0 +O : CanResume: false +O --> DSO : GET 200 +O --> ERROR + +SO : Filename: "" +SO : Size: > 0 +SO : CanResume: false +SO --> DSO : GET: 200 +SO --> ERROR + +DO : Filename: "foo.bar" +DO : Size: 0 +DO : CanResume: false +DO --> DSO : GET 200 +DO --> ERROR + +DSZ : Filename: "foo.bar" +DSZ : Size: > 0 +DSZ : File: empty +DSZ --> DSORZ : HEAD: Range supported +DSZ --> DSOZ : HEAD 405 or Range unsupported + +DSP : Filename: "foo.bar" +DSP : Size: > 0 +DSP : File: partial +DSP --> DSORP : HEAD: Range supported +DSP --> DSOZ : HEAD: 405 or Range unsupported + +DSO : Filename: "foo.bar" +DSO : Size: > 0 +DSO : CanResume: false +DSO --> DSOZ : checkExisting(): File partial|missing +DSO --> [*] : checkExisting(): File complete + +DSOR : Filename: "foo.bar" +DSOR : Size: > 0 +DSOR : CanResume: true +DSOR --> DSORP : CheckLocal: File partial +DSOR --> DSORZ : CheckLocal: File missing + +DSORP : Filename: "foo.bar" +DSORP : Size: > 0 +DSORP : CanResume: true +DSORP : File: partial +DSORP --> Transferring + +DSORZ : Filename: "foo.bar" +DSORZ : Size: > 0 +DSORZ : CanResume: true +DSORZ : File: empty +DSORZ --> Transferring + +DSOZ : Filename: "foo.bar" +DSOZ : Size: > 0 +DSOZ : CanResume: false +DSOZ : File: empty +DSOZ --> Transferring + +Transferring --> [*] +Transferring --> ERROR + +ERROR : Something went wrong +ERROR --> [*] + +@enduml \ No newline at end of file diff --git a/vendor/github.com/cavaliercoder/grab/transfer.go b/vendor/github.com/cavaliercoder/grab/transfer.go new file mode 100644 index 00000000..6fe20556 --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/transfer.go @@ -0,0 +1,81 @@ +package grab + +import ( + "context" + "io" + "sync/atomic" +) + +type transfer struct { + n int64 // must be 64bit aligned on 386 + ctx context.Context + lim RateLimiter + w io.Writer + r io.Reader + b []byte +} + +func newTransfer(ctx context.Context, lim RateLimiter, dst io.Writer, src io.Reader, buf []byte) *transfer { + return &transfer{ + ctx: ctx, + lim: lim, + w: dst, + r: src, + b: buf, + } +} + +// copy behaves similarly to io.CopyBuffer except that it checks for cancelation +// of the given context.Context and reports progress in a thread-safe manner. +func (c *transfer) copy() (written int64, err error) { + if c.b == nil { + c.b = make([]byte, 32*1024) + } + for { + select { + case <-c.ctx.Done(): + err = c.ctx.Err() + return + default: + // keep working + } + if c.lim != nil { + err = c.lim.WaitN(c.ctx, len(c.b)) + if err != nil { + return + } + } + nr, er := c.r.Read(c.b) + if nr > 0 { + nw, ew := c.w.Write(c.b[0:nr]) + if nw > 0 { + written += int64(nw) + atomic.StoreInt64(&c.n, written) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + return written, err +} + +// N returns the number of bytes transferred. +func (c *transfer) N() (n int64) { + if c == nil { + return 0 + } + n = atomic.LoadInt64(&c.n) + return +} diff --git a/vendor/github.com/cavaliercoder/grab/util.go b/vendor/github.com/cavaliercoder/grab/util.go new file mode 100644 index 00000000..1890fe3a --- /dev/null +++ b/vendor/github.com/cavaliercoder/grab/util.go @@ -0,0 +1,89 @@ +package grab + +import ( + "context" + "fmt" + "hash" + "mime" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +// setLastModified sets the last modified timestamp of a local file according to +// the Last-Modified header returned by a remote server. +func setLastModified(resp *http.Response, filename string) error { + // https://tools.ietf.org/html/rfc7232#section-2.2 + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified + header := resp.Header.Get("Last-Modified") + if header == "" { + return nil + } + lastmod, err := time.Parse(http.TimeFormat, header) + if err != nil { + return nil + } + return os.Chtimes(filename, lastmod, lastmod) +} + +// mkdirp creates all missing parent directories for the destination file path. +func mkdirp(path string) error { + dir := filepath.Dir(path) + if fi, err := os.Stat(dir); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("error checking destination directory: %v", err) + } + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("error creating destination directory: %v", err) + } + } else if !fi.IsDir() { + panic("destination path is not directory") + } + return nil +} + +// guessFilename returns a filename for the given http.Response. If none can be +// determined ErrNoFilename is returned. +func guessFilename(resp *http.Response) (string, error) { + filename := resp.Request.URL.Path + if cd := resp.Header.Get("Content-Disposition"); cd != "" { + if _, params, err := mime.ParseMediaType(cd); err == nil { + filename = params["filename"] + } + } + + // sanitize + if filename == "" || strings.HasSuffix(filename, "/") || strings.Contains(filename, "\x00") { + return "", ErrNoFilename + } + + filename = filepath.Base(path.Clean("/" + filename)) + if filename == "" || filename == "." || filename == "/" { + return "", ErrNoFilename + } + + return filename, nil +} + +// checksum returns a hash of the given file, using the given hash algorithm. +func checksum(ctx context.Context, filename string, h hash.Hash) (b []byte, err error) { + var f *os.File + f, err = os.Open(filename) + if err != nil { + return + } + defer func() { + err = f.Close() + }() + + t := newTransfer(ctx, nil, h, f, nil) + if _, err = t.copy(); err != nil { + return + } + + b = h.Sum(nil) + return +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f1ae5f72..fc7ed7b3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -34,6 +34,8 @@ github.com/asdine/storm/index github.com/asdine/storm/internal # github.com/briandowns/spinner v1.7.0 github.com/briandowns/spinner +# github.com/cavaliercoder/grab v2.0.0+incompatible +github.com/cavaliercoder/grab # github.com/containerd/containerd v1.2.4 github.com/containerd/containerd/archive github.com/containerd/containerd/log