update vendor/

vendor/github.com/cavaliercoder/grab/.travis.yml

@@ -0,0 +1,13 @@
+language: go
+
+go:
+  - 1.10.x
+  - 1.9.x
+  - 1.8.x
+  - 1.7.x
+
+script: make check
+
+env:
+  - GOARCH=amd64
+  - GOARCH=386

vendor/github.com/cavaliercoder/grab/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2017 Ryan Armstrong. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/cavaliercoder/grab/Makefile

@@ -0,0 +1,29 @@
+GO = go
+GOGET = $(GO) get -u
+
+all: check lint
+
+check:
+	cd cmd/grab && $(MAKE) -B all
+	$(GO) test -cover -race ./...
+
+install:
+	$(GO) install -v ./...
+
+clean:
+	$(GO) clean -x ./...
+	rm -rvf ./.test*
+
+lint:
+	gofmt -l -e -s . || :
+	go vet . || :
+	golint . || :
+	gocyclo -over 15 . || :
+	misspell ./* || :
+
+deps:
+	$(GOGET) github.com/golang/lint/golint
+	$(GOGET) github.com/fzipp/gocyclo
+	$(GOGET) github.com/client9/misspell/cmd/misspell
+
+.PHONY: all check install clean lint deps

vendor/github.com/cavaliercoder/grab/README.md

@@ -0,0 +1,127 @@
+# grab
+
+[![GoDoc](https://godoc.org/github.com/cavaliercoder/grab?status.svg)](https://godoc.org/github.com/cavaliercoder/grab) [![Build Status](https://travis-ci.org/cavaliercoder/grab.svg?branch=master)](https://travis-ci.org/cavaliercoder/grab) [![Go Report Card](https://goreportcard.com/badge/github.com/cavaliercoder/grab)](https://goreportcard.com/report/github.com/cavaliercoder/grab)
+
+*Downloading the internet, one goroutine at a time!*
+
+	$ go get github.com/cavaliercoder/grab
+
+Grab is a Go package for downloading files from the internet with the following
+rad features:
+
+* Monitor download progress concurrently
+* Auto-resume incomplete downloads
+* Guess filename from content header or URL path
+* Safely cancel downloads using context.Context
+* Validate downloads using checksums
+* Download batches of files concurrently
+* Apply rate limiters
+
+Requires Go v1.7+
+
+## Example
+
+The following example downloads a PDF copy of the free eBook, "An Introduction
+to Programming in Go" into the current working directory.
+
+```go
+resp, err := grab.Get(".", "http://www.golang-book.com/public/pdf/gobook.pdf")
+if err != nil {
+	log.Fatal(err)
+}
+
+fmt.Println("Download saved to", resp.Filename)
+```
+
+The following, more complete example allows for more granular control and
+periodically prints the download progress until it is complete.
+
+The second time you run the example, it will auto-resume the previous download
+and exit sooner.
+
+```go
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/cavaliercoder/grab"
+)
+
+func main() {
+	// create client
+	client := grab.NewClient()
+	req, _ := grab.NewRequest(".", "http://www.golang-book.com/public/pdf/gobook.pdf")
+
+	// start download
+	fmt.Printf("Downloading %v...\n", req.URL())
+	resp := client.Do(req)
+	fmt.Printf("  %v\n", resp.HTTPResponse.Status)
+
+	// start UI loop
+	t := time.NewTicker(500 * time.Millisecond)
+	defer t.Stop()
+
+Loop:
+	for {
+		select {
+		case <-t.C:
+			fmt.Printf("  transferred %v / %v bytes (%.2f%%)\n",
+				resp.BytesComplete(),
+				resp.Size,
+				100*resp.Progress())
+
+		case <-resp.Done:
+			// download is complete
+			break Loop
+		}
+	}
+
+	// check for errors
+	if err := resp.Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "Download failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Download saved to ./%v \n", resp.Filename)
+
+	// Output:
+	// Downloading http://www.golang-book.com/public/pdf/gobook.pdf...
+	//   200 OK
+	//   transferred 42970 / 2893557 bytes (1.49%)
+	//   transferred 1207474 / 2893557 bytes (41.73%)
+	//   transferred 2758210 / 2893557 bytes (95.32%)
+	// Download saved to ./gobook.pdf
+}
+```
+
+## Design trade-offs
+
+The primary use case for Grab is to concurrently downloading thousands of large
+files from remote file repositories where the remote files are immutable.
+Examples include operating system package repositories or ISO libraries.
+
+Grab aims to provide robust, sane defaults. These are usually determined using
+the HTTP specifications, or by mimicking the behavior of common web clients like
+cURL, wget and common web browsers.
+
+Grab aims to be stateless. The only state that exists is the remote files you
+wish to download and the local copy which may be completed, partially completed
+or not yet created. The advantage to this is that the local file system is not
+cluttered unnecessarily with addition state files (like a `.crdownload` file).
+The disadvantage of this approach is that grab must make assumptions about the
+local and remote state; specifically, that they have not been modified by
+another program.
+
+If the local or remote file are modified outside of grab, and you download the
+file again with resuming enabled, the local file will likely become corrupted.
+In this case, you might consider making remote files immutable, or disabling
+resume.
+
+Grab aims to enable best-in-class functionality for more complex features
+through extensible interfaces, rather than reimplementation. For example,
+you can provide your own Hash algorithm to compute file checksums, or your
+own rate limiter implementation (with all the associated trade-offs) to rate
+limit downloads.

vendor/github.com/cavaliercoder/grab/client.go

@@ -0,0 +1,506 @@
+package grab
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+// A Client is a file download client.
+//
+// Clients are safe for concurrent use by multiple goroutines.
+type Client struct {
+	// HTTPClient specifies the http.Client which will be used for communicating
+	// with the remote server during the file transfer.
+	HTTPClient *http.Client
+
+	// UserAgent specifies the User-Agent string which will be set in the
+	// headers of all requests made by this client.
+	//
+	// The user agent string may be overridden in the headers of each request.
+	UserAgent string
+
+	// BufferSize specifies the size in bytes of the buffer that is used for
+	// transferring all requested files. Larger buffers may result in faster
+	// throughput but will use more memory and result in less frequent updates
+	// to the transfer progress statistics. The BufferSize of each request can
+	// be overridden on each Request object. Default: 32KB.
+	BufferSize int
+}
+
+// NewClient returns a new file download Client, using default configuration.
+func NewClient() *Client {
+	return &Client{
+		UserAgent: "grab",
+		HTTPClient: &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+			},
+		},
+	}
+}
+
+// DefaultClient is the default client and is used by all Get convenience
+// functions.
+var DefaultClient = NewClient()
+
+// Do sends a file transfer request and returns a file transfer response,
+// following policy (e.g. redirects, cookies, auth) as configured on the
+// client's HTTPClient.
+//
+// Like http.Get, Do blocks while the transfer is initiated, but returns as soon
+// as the transfer has started transferring in a background goroutine, or if it
+// failed early.
+//
+// An error is returned via Response.Err if caused by client policy (such as
+// CheckRedirect), or if there was an HTTP protocol or IO error. Response.Err
+// will block the caller until the transfer is completed, successfully or
+// otherwise.
+func (c *Client) Do(req *Request) *Response {
+	// cancel will be called on all code-paths via closeResponse
+	ctx, cancel := context.WithCancel(req.Context())
+	resp := &Response{
+		Request:    req,
+		Start:      time.Now(),
+		Done:       make(chan struct{}, 0),
+		Filename:   req.Filename,
+		ctx:        ctx,
+		cancel:     cancel,
+		bufferSize: req.BufferSize,
+	}
+	if resp.bufferSize == 0 {
+		// default to Client.BufferSize
+		resp.bufferSize = c.BufferSize
+	}
+
+	// Run state-machine while caller is blocked to initialize the file transfer.
+	// Must never transition to the copyFile state - this happens next in another
+	// goroutine.
+	c.run(resp, c.statFileInfo)
+
+	// Run copyFile in a new goroutine. copyFile will no-op if the transfer is
+	// already complete or failed.
+	go c.run(resp, c.copyFile)
+	return resp
+}
+
+// DoChannel executes all requests sent through the given Request channel, one
+// at a time, until it is closed by another goroutine. The caller is blocked
+// until the Request channel is closed and all transfers have completed. All
+// responses are sent through the given Response channel as soon as they are
+// received from the remote servers and can be used to track the progress of
+// each download.
+//
+// Slow Response receivers will cause a worker to block and therefore delay the
+// start of the transfer for an already initiated connection - potentially
+// causing a server timeout. It is the caller's responsibility to ensure a
+// sufficient buffer size is used for the Response channel to prevent this.
+//
+// If an error occurs during any of the file transfers it will be accessible via
+// the associated Response.Err function.
+func (c *Client) DoChannel(reqch <-chan *Request, respch chan<- *Response) {
+	// TODO: enable cancelling of batch jobs
+	for req := range reqch {
+		resp := c.Do(req)
+		respch <- resp
+		<-resp.Done
+	}
+}
+
+// DoBatch executes all the given requests using the given number of concurrent
+// workers. Control is passed back to the caller as soon as the workers are
+// initiated.
+//
+// If the requested number of workers is less than one, a worker will be created
+// for every request. I.e. all requests will be executed concurrently.
+//
+// If an error occurs during any of the file transfers it will be accessible via
+// call to the associated Response.Err.
+//
+// The returned Response channel is closed only after all of the given Requests
+// have completed, successfully or otherwise.
+func (c *Client) DoBatch(workers int, requests ...*Request) <-chan *Response {
+	if workers < 1 {
+		workers = len(requests)
+	}
+	reqch := make(chan *Request, len(requests))
+	respch := make(chan *Response, len(requests))
+	wg := sync.WaitGroup{}
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			c.DoChannel(reqch, respch)
+			wg.Done()
+		}()
+	}
+
+	// queue requests
+	go func() {
+		for _, req := range requests {
+			reqch <- req
+		}
+		close(reqch)
+		wg.Wait()
+		close(respch)
+	}()
+	return respch
+}
+
+// An stateFunc is an action that mutates the state of a Response and returns
+// the next stateFunc to be called.
+type stateFunc func(*Response) stateFunc
+
+// run calls the given stateFunc function and all subsequent returned stateFuncs
+// until a stateFunc returns nil or the Response.ctx is canceled. Each stateFunc
+// should mutate the state of the given Response until it has completed
+// downloading or failed.
+func (c *Client) run(resp *Response, f stateFunc) {
+	for {
+		select {
+		case <-resp.ctx.Done():
+			if resp.IsComplete() {
+				return
+			}
+			resp.err = resp.ctx.Err()
+			f = c.closeResponse
+
+		default:
+			// keep working
+		}
+		if f = f(resp); f == nil {
+			return
+		}
+	}
+}
+
+// statFileInfo retrieves FileInfo for any local file matching
+// Response.Filename.
+//
+// If the file does not exist, is a directory, or its name is unknown the next
+// stateFunc is headRequest.
+//
+// If the file exists, Response.fi is set and the next stateFunc is
+// validateLocal.
+//
+// If an error occurs, the next stateFunc is closeResponse.
+func (c *Client) statFileInfo(resp *Response) stateFunc {
+	if resp.Filename == "" {
+		return c.headRequest
+	}
+	fi, err := os.Stat(resp.Filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return c.headRequest
+		}
+		resp.err = err
+		return c.closeResponse
+	}
+	if fi.IsDir() {
+		resp.Filename = ""
+		return c.headRequest
+	}
+	resp.fi = fi
+	return c.validateLocal
+}
+
+// validateLocal compares a local copy of the downloaded file to the remote
+// file.
+//
+// An error is returned if the local file is larger than the remote file, or
+// Request.SkipExisting is true.
+//
+// If the existing file matches the length of the remote file, the next
+// stateFunc is checksumFile.
+//
+// If the local file is smaller than the remote file and the remote server is
+// known to support ranged requests, the next stateFunc is getRequest.
+func (c *Client) validateLocal(resp *Response) stateFunc {
+	if resp.Request.SkipExisting {
+		resp.err = ErrFileExists
+		return c.closeResponse
+	}
+
+	// determine expected file size
+	size := resp.Request.Size
+	if size == 0 && resp.HTTPResponse != nil {
+		size = resp.HTTPResponse.ContentLength
+	}
+	if size == 0 {
+		return c.headRequest
+	}
+
+	if size == resp.fi.Size() {
+		resp.DidResume = true
+		resp.bytesResumed = resp.fi.Size()
+		return c.checksumFile
+	}
+
+	if resp.Request.NoResume {
+		return c.getRequest
+	}
+
+	if size < resp.fi.Size() {
+		resp.err = ErrBadLength
+		return c.closeResponse
+	}
+
+	if resp.CanResume {
+		resp.Request.HTTPRequest.Header.Set(
+			"Range",
+			fmt.Sprintf("bytes=%d-", resp.fi.Size()))
+		resp.DidResume = true
+		resp.bytesResumed = resp.fi.Size()
+		return c.getRequest
+	}
+	return c.headRequest
+}
+
+func (c *Client) checksumFile(resp *Response) stateFunc {
+	if resp.Request.hash == nil {
+		return c.closeResponse
+	}
+	if resp.Filename == "" {
+		panic("filename not set")
+	}
+	req := resp.Request
+
+	// compare checksum
+	var sum []byte
+	sum, resp.err = checksum(req.Context(), resp.Filename, req.hash)
+	if resp.err != nil {
+		return c.closeResponse
+	}
+	if !bytes.Equal(sum, req.checksum) {
+		resp.err = ErrBadChecksum
+		if req.deleteOnError {
+			if err := os.Remove(resp.Filename); err != nil {
+				// err should be os.PathError and include file path
+				resp.err = fmt.Errorf(
+					"cannot remove downloaded file with checksum mismatch: %v",
+					err)
+			}
+		}
+	}
+	return c.closeResponse
+}
+
+// doHTTPRequest sends a HTTP Request and returns the response
+func (c *Client) doHTTPRequest(req *http.Request) (*http.Response, error) {
+	if c.UserAgent != "" && req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", c.UserAgent)
+	}
+	return c.HTTPClient.Do(req)
+}
+
+func (c *Client) headRequest(resp *Response) stateFunc {
+	if resp.optionsKnown {
+		return c.getRequest
+	}
+	resp.optionsKnown = true
+
+	if resp.Request.NoResume {
+		return c.getRequest
+	}
+
+	if resp.Filename != "" && resp.fi == nil {
+		// destination path is already known and does not exist
+		return c.getRequest
+	}
+
+	hreq := new(http.Request)
+	*hreq = *resp.Request.HTTPRequest
+	hreq.Method = "HEAD"
+
+	resp.HTTPResponse, resp.err = c.doHTTPRequest(hreq)
+	if resp.err != nil {
+		return c.closeResponse
+	}
+	resp.HTTPResponse.Body.Close()
+
+	if resp.HTTPResponse.StatusCode != http.StatusOK {
+		return c.getRequest
+	}
+
+	return c.readResponse
+}
+
+func (c *Client) getRequest(resp *Response) stateFunc {
+	resp.HTTPResponse, resp.err = c.doHTTPRequest(resp.Request.HTTPRequest)
+	if resp.err != nil {
+		return c.closeResponse
+	}
+
+	// check status code
+	if !resp.Request.IgnoreBadStatusCodes {
+		if resp.HTTPResponse.StatusCode < 200 || resp.HTTPResponse.StatusCode > 299 {
+			resp.err = StatusCodeError(resp.HTTPResponse.StatusCode)
+			return c.closeResponse
+		}
+	}
+
+	return c.readResponse
+}
+
+func (c *Client) readResponse(resp *Response) stateFunc {
+	if resp.HTTPResponse == nil {
+		panic("Response.HTTPResponse is not ready")
+	}
+
+	// check expected size
+	resp.Size = resp.bytesResumed + resp.HTTPResponse.ContentLength
+	if resp.HTTPResponse.ContentLength > 0 && resp.Request.Size > 0 {
+		if resp.Request.Size != resp.Size {
+			resp.err = ErrBadLength
+			return c.closeResponse
+		}
+	}
+
+	// check filename
+	if resp.Filename == "" {
+		filename, err := guessFilename(resp.HTTPResponse)
+		if err != nil {
+			resp.err = err
+			return c.closeResponse
+		}
+		// Request.Filename will be empty or a directory
+		resp.Filename = filepath.Join(resp.Request.Filename, filename)
+	}
+
+	if resp.requestMethod() == "HEAD" {
+		if resp.HTTPResponse.Header.Get("Accept-Ranges") == "bytes" {
+			resp.CanResume = true
+		}
+		return c.statFileInfo
+	}
+	return c.openWriter
+}
+
+// openWriter opens the destination file for writing and seeks to the location
+// from whence the file transfer will resume.
+//
+// Requires that Response.Filename and resp.DidResume are already be set.
+func (c *Client) openWriter(resp *Response) stateFunc {
+	if !resp.Request.NoCreateDirectories {
+		resp.err = mkdirp(resp.Filename)
+		if resp.err != nil {
+			return c.closeResponse
+		}
+	}
+
+	// compute write flags
+	flag := os.O_CREATE | os.O_WRONLY
+	if resp.fi != nil {
+		if resp.DidResume {
+			flag = os.O_APPEND | os.O_WRONLY
+		} else {
+			flag = os.O_TRUNC | os.O_WRONLY
+		}
+	}
+
+	// open file
+	f, err := os.OpenFile(resp.Filename, flag, 0644)
+	if err != nil {
+		resp.err = err
+		return c.closeResponse
+	}
+	resp.writer = f
+
+	// seek to start or end
+	whence := os.SEEK_SET
+	if resp.bytesResumed > 0 {
+		whence = os.SEEK_END
+	}
+	_, resp.err = f.Seek(0, whence)
+	if resp.err != nil {
+		return c.closeResponse
+	}
+
+	// init transfer
+	if resp.bufferSize < 1 {
+		resp.bufferSize = 32 * 1024
+	}
+	b := make([]byte, resp.bufferSize)
+	resp.transfer = newTransfer(
+		resp.Request.Context(),
+		resp.Request.RateLimiter,
+		resp.writer,
+		resp.HTTPResponse.Body,
+		b)
+
+	// next step is copyFile, but this will be called later in another goroutine
+	return nil
+}
+
+// copy transfers content for a HTTP connection established via Client.do()
+func (c *Client) copyFile(resp *Response) stateFunc {
+	if resp.IsComplete() {
+		return nil
+	}
+
+	// run BeforeCopy hook
+	if f := resp.Request.BeforeCopy; f != nil {
+		resp.err = f(resp)
+		if resp.err != nil {
+			return c.closeResponse
+		}
+	}
+
+	if resp.transfer == nil {
+		panic("developer error: Response.transfer is not initialized")
+	}
+	go resp.watchBps()
+	_, resp.err = resp.transfer.copy()
+	if resp.err != nil {
+		return c.closeResponse
+	}
+	closeWriter(resp)
+
+	// set timestamp
+	if !resp.Request.IgnoreRemoteTime {
+		resp.err = setLastModified(resp.HTTPResponse, resp.Filename)
+		if resp.err != nil {
+			return c.closeResponse
+		}
+	}
+
+	// run AfterCopy hook
+	if f := resp.Request.AfterCopy; f != nil {
+		resp.err = f(resp)
+		if resp.err != nil {
+			return c.closeResponse
+		}
+	}
+
+	return c.checksumFile
+}
+
+func closeWriter(resp *Response) {
+	if resp.writer != nil {
+		resp.writer.Close()
+		resp.writer = nil
+	}
+}
+
+// close finalizes the Response
+func (c *Client) closeResponse(resp *Response) stateFunc {
+	if resp.IsComplete() {
+		panic("response already closed")
+	}
+
+	resp.fi = nil
+	closeWriter(resp)
+	resp.closeResponseBody()
+
+	resp.End = time.Now()
+	close(resp.Done)
+	if resp.cancel != nil {
+		resp.cancel()
+	}
+
+	return nil
+}

vendor/github.com/cavaliercoder/grab/doc.go

@@ -0,0 +1,63 @@
+/*
+Package grab provides a HTTP download manager implementation.
+
+Get is the most simple way to download a file:
+
+	resp, err := grab.Get("/tmp", "http://example.com/example.zip")
+	// ...
+
+Get will download the given URL and save it to the given destination directory.
+The destination filename will be determined automatically by grab using
+Content-Disposition headers returned by the remote server, or by inspecting the
+requested URL path.
+
+An empty destination string or "." means the transfer will be stored in the
+current working directory.
+
+If a destination file already exists, grab will assume it is a complete or
+partially complete download of the requested file. If the remote server supports
+resuming interrupted downloads, grab will resume downloading from the end of the
+partial file. If the server does not support resumed downloads, the file will be
+retransferred in its entirety. If the file is already complete, grab will return
+successfully.
+
+For control over the HTTP client, destination path, auto-resume, checksum
+validation and other settings, create a Client:
+
+	client := grab.NewClient()
+	client.HTTPClient.Transport.DisableCompression = true
+
+	req, err := grab.NewRequest("/tmp", "http://example.com/example.zip")
+	// ...
+	req.NoResume = true
+	req.HTTPRequest.Header.Set("Authorization", "Basic YWxhZGRpbjpvcGVuc2VzYW1l")
+
+	resp := client.Do(req)
+	// ...
+
+You can monitor the progress of downloads while they are transferring:
+
+	client := grab.NewClient()
+	req, err := grab.NewRequest("", "http://example.com/example.zip")
+	// ...
+	resp := client.Do(req)
+
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			fmt.Printf("%.02f%% complete\n", resp.Progress())
+
+		case <-resp.Done:
+			if err := resp.Err(); err != nil {
+				// ...
+			}
+
+			// ...
+			return
+		}
+	}
+*/
+package grab

vendor/github.com/cavaliercoder/grab/error.go

@@ -0,0 +1,42 @@
+package grab
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+)
+
+var (
+	// ErrBadLength indicates that the server response or an existing file does
+	// not match the expected content length.
+	ErrBadLength = errors.New("bad content length")
+
+	// ErrBadChecksum indicates that a downloaded file failed to pass checksum
+	// validation.
+	ErrBadChecksum = errors.New("checksum mismatch")
+
+	// ErrNoFilename indicates that a reasonable filename could not be
+	// automatically determined using the URL or response headers from a server.
+	ErrNoFilename = errors.New("no filename could be determined")
+
+	// ErrNoTimestamp indicates that a timestamp could not be automatically
+	// determined using the response headers from the remote server.
+	ErrNoTimestamp = errors.New("no timestamp could be determined for the remote file")
+
+	// ErrFileExists indicates that the destination path already exists.
+	ErrFileExists = errors.New("file exists")
+)
+
+// StatusCodeError indicates that the server response had a status code that
+// was not in the 200-299 range (after following any redirects).
+type StatusCodeError int
+
+func (err StatusCodeError) Error() string {
+	return fmt.Sprintf("server returned %d %s", err, http.StatusText(int(err)))
+}
+
+// IsStatusCodeError returns true if the given error is of type StatusCodeError.
+func IsStatusCodeError(err error) bool {
+	_, ok := err.(StatusCodeError)
+	return ok
+}

vendor/github.com/cavaliercoder/grab/grab.go

@@ -0,0 +1,64 @@
+package grab
+
+import (
+	"fmt"
+	"os"
+)
+
+// Get sends a HTTP request and downloads the content of the requested URL to
+// the given destination file path. The caller is blocked until the download is
+// completed, successfully or otherwise.
+//
+// An error is returned if caused by client policy (such as CheckRedirect), or
+// if there was an HTTP protocol or IO error.
+//
+// For non-blocking calls or control over HTTP client headers, redirect policy,
+// and other settings, create a Client instead.
+func Get(dst, urlStr string) (*Response, error) {
+	req, err := NewRequest(dst, urlStr)
+	if err != nil {
+		return nil, err
+	}
+
+	resp := DefaultClient.Do(req)
+	return resp, resp.Err()
+}
+
+// GetBatch sends multiple HTTP requests and downloads the content of the
+// requested URLs to the given destination directory using the given number of
+// concurrent worker goroutines.
+//
+// The Response for each requested URL is sent through the returned Response
+// channel, as soon as a worker receives a response from the remote server. The
+// Response can then be used to track the progress of the download while it is
+// in progress.
+//
+// The returned Response channel will be closed by Grab, only once all downloads
+// have completed or failed.
+//
+// If an error occurs during any download, it will be available via call to the
+// associated Response.Err.
+//
+// For control over HTTP client headers, redirect policy, and other settings,
+// create a Client instead.
+func GetBatch(workers int, dst string, urlStrs ...string) (<-chan *Response, error) {
+	fi, err := os.Stat(dst)
+	if err != nil {
+		return nil, err
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("destination is not a directory")
+	}
+
+	reqs := make([]*Request, len(urlStrs))
+	for i := 0; i < len(urlStrs); i++ {
+		req, err := NewRequest(dst, urlStrs[i])
+		if err != nil {
+			return nil, err
+		}
+		reqs[i] = req
+	}
+
+	ch := DefaultClient.DoBatch(workers, reqs...)
+	return ch, nil
+}

vendor/github.com/cavaliercoder/grab/rate_limiter.go

@@ -0,0 +1,12 @@
+package grab
+
+import "context"
+
+// RateLimiter is an interface that must be satisfied by any third-party rate
+// limiters that may be used to limit download transfer speeds.
+//
+// A recommended token bucket implementation can be found at
+// https://godoc.org/golang.org/x/time/rate#Limiter.
+type RateLimiter interface {
+	WaitN(ctx context.Context, n int) (err error)
+}

vendor/github.com/cavaliercoder/grab/request.go

@@ -0,0 +1,172 @@
+package grab
+
+import (
+	"context"
+	"hash"
+	"net/http"
+	"net/url"
+)
+
+// A Hook is a user provided callback function that can be called by grab at
+// various stages of a requests lifecycle. If a hook returns an error, the
+// associated request is canceled and the same error is returned on the Response
+// object.
+//
+// Hook functions are called synchronously and should never block unnecessarily.
+// Response methods that block until a download is complete, such as
+// Response.Err, Response.Cancel or Response.Wait will deadlock. To cancel a
+// download from a callback, simply return a non-nil error.
+type Hook func(*Response) error
+
+// A Request represents an HTTP file transfer request to be sent by a Client.
+type Request struct {
+	// Label is an arbitrary string which may used to label a Request with a
+	// user friendly name.
+	Label string
+
+	// Tag is an arbitrary interface which may be used to relate a Request to
+	// other data.
+	Tag interface{}
+
+	// HTTPRequest specifies the http.Request to be sent to the remote server to
+	// initiate a file transfer. It includes request configuration such as URL,
+	// protocol version, HTTP method, request headers and authentication.
+	HTTPRequest *http.Request
+
+	// Filename specifies the path where the file transfer will be stored in
+	// local storage. If Filename is empty or a directory, the true Filename will
+	// be resolved using Content-Disposition headers or the request URL.
+	//
+	// An empty string means the transfer will be stored in the current working
+	// directory.
+	Filename string
+
+	// SkipExisting specifies that ErrFileExists should be returned if the
+	// destination path already exists. The existing file will not be checked for
+	// completeness.
+	SkipExisting bool
+
+	// NoResume specifies that a partially completed download will be restarted
+	// without attempting to resume any existing file. If the download is already
+	// completed in full, it will not be restarted.
+	NoResume bool
+
+	// NoCreateDirectories specifies that any missing directories in the given
+	// Filename path should not be created automatically, if they do not already
+	// exist.
+	NoCreateDirectories bool
+
+	// IgnoreBadStatusCodes specifies that grab should accept any status code in
+	// the response from the remote server. Otherwise, grab expects the response
+	// status code to be within the 2XX range (after following redirects).
+	IgnoreBadStatusCodes bool
+
+	// IgnoreRemoteTime specifies that grab should not attempt to set the
+	// timestamp of the local file to match the remote file.
+	IgnoreRemoteTime bool
+
+	// Size specifies the expected size of the file transfer if known. If the
+	// server response size does not match, the transfer is cancelled and
+	// ErrBadLength returned.
+	Size int64
+
+	// BufferSize specifies the size in bytes of the buffer that is used for
+	// transferring the requested file. Larger buffers may result in faster
+	// throughput but will use more memory and result in less frequent updates
+	// to the transfer progress statistics. If a RateLimiter is configured,
+	// BufferSize should be much lower than the rate limit. Default: 32KB.
+	BufferSize int
+
+	// RateLimiter allows the transfer rate of a download to be limited. The given
+	// Request.BufferSize determines how frequently the RateLimiter will be
+	// polled.
+	RateLimiter RateLimiter
+
+	// BeforeCopy is a user provided callback that is called immediately before
+	// a request starts downloading. If BeforeCopy returns an error, the request
+	// is cancelled and the same error is returned on the Response object.
+	BeforeCopy Hook
+
+	// AfterCopy is a user provided callback that is called immediately after a
+	// request has finished downloading, before checksum validation and closure.
+	// This hook is only called if the transfer was successful. If AfterCopy
+	// returns an error, the request is canceled and the same error is returned on
+	// the Response object.
+	AfterCopy Hook
+
+	// hash, checksum and deleteOnError - set via SetChecksum.
+	hash          hash.Hash
+	checksum      []byte
+	deleteOnError bool
+
+	// Context for cancellation and timeout - set via WithContext
+	ctx context.Context
+}
+
+// NewRequest returns a new file transfer Request suitable for use with
+// Client.Do.
+func NewRequest(dst, urlStr string) (*Request, error) {
+	if dst == "" {
+		dst = "."
+	}
+	req, err := http.NewRequest("GET", urlStr, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &Request{
+		HTTPRequest: req,
+		Filename:    dst,
+	}, nil
+}
+
+// Context returns the request's context. To change the context, use
+// WithContext.
+//
+// The returned context is always non-nil; it defaults to the background
+// context.
+//
+// The context controls cancelation.
+func (r *Request) Context() context.Context {
+	if r.ctx != nil {
+		return r.ctx
+	}
+
+	return context.Background()
+}
+
+// WithContext returns a shallow copy of r with its context changed
+// to ctx. The provided ctx must be non-nil.
+func (r *Request) WithContext(ctx context.Context) *Request {
+	if ctx == nil {
+		panic("nil context")
+	}
+	r2 := new(Request)
+	*r2 = *r
+	r2.ctx = ctx
+	r2.HTTPRequest = r2.HTTPRequest.WithContext(ctx)
+	return r2
+}
+
+// URL returns the URL to be downloaded.
+func (r *Request) URL() *url.URL {
+	return r.HTTPRequest.URL
+}
+
+// SetChecksum sets the desired hashing algorithm and checksum value to validate
+// a downloaded file. Once the download is complete, the given hashing algorithm
+// will be used to compute the actual checksum of the downloaded file. If the
+// checksums do not match, an error will be returned by the associated
+// Response.Err method.
+//
+// If deleteOnError is true, the downloaded file will be deleted automatically
+// if it fails checksum validation.
+//
+// To prevent corruption of the computed checksum, the given hash must not be
+// used by any other request or goroutines.
+//
+// To disable checksum validation, call SetChecksum with a nil hash.
+func (r *Request) SetChecksum(h hash.Hash, sum []byte, deleteOnError bool) {
+	r.hash = h
+	r.checksum = sum
+	r.deleteOnError = deleteOnError
+}

vendor/github.com/cavaliercoder/grab/response.go

@@ -0,0 +1,224 @@
+package grab
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+)
+
+// Response represents the response to a completed or in-progress download
+// request.
+//
+// A response may be returned as soon a HTTP response is received from a remote
+// server, but before the body content has started transferring.
+//
+// All Response method calls are thread-safe.
+type Response struct {
+	// The Request that was submitted to obtain this Response.
+	Request *Request
+
+	// HTTPResponse represents the HTTP response received from an HTTP request.
+	//
+	// The response Body should not be used as it will be consumed and closed by
+	// grab.
+	HTTPResponse *http.Response
+
+	// Filename specifies the path where the file transfer is stored in local
+	// storage.
+	Filename string
+
+	// Size specifies the total expected size of the file transfer.
+	Size int64
+
+	// Start specifies the time at which the file transfer started.
+	Start time.Time
+
+	// End specifies the time at which the file transfer completed.
+	//
+	// This will return zero until the transfer has completed.
+	End time.Time
+
+	// CanResume specifies that the remote server advertised that it can resume
+	// previous downloads, as the 'Accept-Ranges: bytes' header is set.
+	CanResume bool
+
+	// DidResume specifies that the file transfer resumed a previously incomplete
+	// transfer.
+	DidResume bool
+
+	// Done is closed once the transfer is finalized, either successfully or with
+	// errors. Errors are available via Response.Err
+	Done chan struct{}
+
+	// ctx is a Context that controls cancelation of an inprogress transfer
+	ctx context.Context
+
+	// cancel is a cancel func that can be used to cancel the context of this
+	// Response.
+	cancel context.CancelFunc
+
+	// fi is the FileInfo for the destination file if it already existed before
+	// transfer started.
+	fi os.FileInfo
+
+	// optionsKnown indicates that a HEAD request has been completed and the
+	// capabilities of the remote server are known.
+	optionsKnown bool
+
+	// writer is the file handle used to write the downloaded file to local
+	// storage
+	writer io.WriteCloser
+
+	// bytesCompleted specifies the number of bytes which were already
+	// transferred before this transfer began.
+	bytesResumed int64
+
+	// transfer is responsible for copying data from the remote server to a local
+	// file, tracking progress and allowing for cancelation.
+	transfer *transfer
+
+	// bytesPerSecond specifies the number of bytes that have been transferred in
+	// the last 1-second window.
+	bytesPerSecond   float64
+	bytesPerSecondMu sync.Mutex
+
+	// bufferSize specifies the size in bytes of the transfer buffer.
+	bufferSize int
+
+	// Error contains any error that may have occurred during the file transfer.
+	// This should not be read until IsComplete returns true.
+	err error
+}
+
+// IsComplete returns true if the download has completed. If an error occurred
+// during the download, it can be returned via Err.
+func (c *Response) IsComplete() bool {
+	select {
+	case <-c.Done:
+		return true
+	default:
+		return false
+	}
+}
+
+// Cancel cancels the file transfer by canceling the underlying Context for
+// this Response. Cancel blocks until the transfer is closed and returns any
+// error - typically context.Canceled.
+func (c *Response) Cancel() error {
+	c.cancel()
+	return c.Err()
+}
+
+// Wait blocks until the download is completed.
+func (c *Response) Wait() {
+	<-c.Done
+}
+
+// Err blocks the calling goroutine until the underlying file transfer is
+// completed and returns any error that may have occurred. If the download is
+// already completed, Err returns immediately.
+func (c *Response) Err() error {
+	<-c.Done
+	return c.err
+}
+
+// BytesComplete returns the total number of bytes which have been copied to
+// the destination, including any bytes that were resumed from a previous
+// download.
+func (c *Response) BytesComplete() int64 {
+	return c.bytesResumed + c.transfer.N()
+}
+
+// BytesPerSecond returns the number of bytes transferred in the last second. If
+// the download is already complete, the average bytes/sec for the life of the
+// download is returned.
+func (c *Response) BytesPerSecond() float64 {
+	if c.IsComplete() {
+		return float64(c.transfer.N()) / c.Duration().Seconds()
+	}
+	c.bytesPerSecondMu.Lock()
+	defer c.bytesPerSecondMu.Unlock()
+	return c.bytesPerSecond
+}
+
+// Progress returns the ratio of total bytes that have been downloaded. Multiply
+// the returned value by 100 to return the percentage completed.
+func (c *Response) Progress() float64 {
+	if c.Size == 0 {
+		return 0
+	}
+	return float64(c.BytesComplete()) / float64(c.Size)
+}
+
+// Duration returns the duration of a file transfer. If the transfer is in
+// process, the duration will be between now and the start of the transfer. If
+// the transfer is complete, the duration will be between the start and end of
+// the completed transfer process.
+func (c *Response) Duration() time.Duration {
+	if c.IsComplete() {
+		return c.End.Sub(c.Start)
+	}
+
+	return time.Now().Sub(c.Start)
+}
+
+// ETA returns the estimated time at which the the download will complete, given
+// the current BytesPerSecond. If the transfer has already completed, the actual
+// end time will be returned.
+func (c *Response) ETA() time.Time {
+	if c.IsComplete() {
+		return c.End
+	}
+	bt := c.BytesComplete()
+	bps := c.BytesPerSecond()
+	if bps == 0 {
+		return time.Time{}
+	}
+	secs := float64(c.Size-bt) / bps
+	return time.Now().Add(time.Duration(secs) * time.Second)
+}
+
+// watchBps watches the progress of a transfer and maintains statistics.
+func (c *Response) watchBps() {
+	var prev int64
+	then := c.Start
+
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-c.Done:
+			return
+
+		case now := <-t.C:
+			d := now.Sub(then)
+			then = now
+
+			cur := c.transfer.N()
+			bs := cur - prev
+			prev = cur
+
+			c.bytesPerSecondMu.Lock()
+			c.bytesPerSecond = float64(bs) / d.Seconds()
+			c.bytesPerSecondMu.Unlock()
+		}
+	}
+}
+
+func (c *Response) requestMethod() string {
+	if c == nil || c.HTTPResponse == nil || c.HTTPResponse.Request == nil {
+		return ""
+	}
+	return c.HTTPResponse.Request.Method
+}
+
+func (c *Response) closeResponseBody() error {
+	if c.HTTPResponse == nil || c.HTTPResponse.Body == nil {
+		return nil
+	}
+	return c.HTTPResponse.Body.Close()
+}

vendor/github.com/cavaliercoder/grab/states.wsd

@@ -0,0 +1,111 @@
+@startuml
+title Grab transfer state
+
+legend
+| # | Meaning |
+| D | Destination path known |
+| S | File size known |
+| O | Server options known (Accept-Ranges) |
+| R | Resume supported (Accept-Ranges) |
+| Z | Local file empty or missing |
+| P | Local file partially complete |
+endlegend
+
+[*] --> Empty
+[*] --> D
+[*] --> S
+[*] --> DS
+
+Empty : Filename: ""
+Empty : Size: 0
+Empty --> O : HEAD: Method not allowed
+Empty --> DSO : HEAD: Range not supported
+Empty --> DSOR : HEAD: Range supported
+
+DS : Filename: "foo.bar"
+DS : Size: > 0
+DS --> DSZ : checkExisting(): File missing
+DS --> DSP : checkExisting(): File partial
+DS --> [*] : checkExisting(): File complete
+DS --> ERROR
+
+S : Filename: ""
+S : Size: > 0
+S --> SO : HEAD: Method not allowed
+S --> DSO : HEAD: Range not supported
+S --> DSOR : HEAD: Range supported
+
+D : Filename: "foo.bar"
+D : Size: 0
+D --> DO : HEAD: Method not allowed
+D --> DSO : HEAD: Range not supported
+D --> DSOR : HEAD: Range supported
+
+
+O : Filename: ""
+O : Size: 0
+O : CanResume: false
+O --> DSO : GET 200
+O --> ERROR
+
+SO : Filename: ""
+SO : Size: > 0
+SO : CanResume: false
+SO --> DSO : GET: 200
+SO --> ERROR
+
+DO : Filename: "foo.bar"
+DO : Size: 0
+DO : CanResume: false
+DO --> DSO : GET 200
+DO --> ERROR
+
+DSZ : Filename: "foo.bar"
+DSZ : Size: > 0
+DSZ : File: empty
+DSZ --> DSORZ : HEAD: Range supported
+DSZ --> DSOZ : HEAD 405 or Range unsupported
+
+DSP : Filename: "foo.bar"
+DSP : Size: > 0
+DSP : File: partial
+DSP --> DSORP : HEAD: Range supported
+DSP --> DSOZ : HEAD: 405 or Range unsupported
+
+DSO : Filename: "foo.bar"
+DSO : Size: > 0
+DSO : CanResume: false
+DSO --> DSOZ : checkExisting(): File partial|missing
+DSO --> [*] : checkExisting(): File complete
+
+DSOR : Filename: "foo.bar"
+DSOR : Size: > 0
+DSOR : CanResume: true
+DSOR --> DSORP : CheckLocal: File partial
+DSOR --> DSORZ : CheckLocal: File missing
+
+DSORP : Filename: "foo.bar"
+DSORP : Size: > 0
+DSORP : CanResume: true
+DSORP : File: partial
+DSORP --> Transferring
+
+DSORZ : Filename: "foo.bar"
+DSORZ : Size: > 0
+DSORZ : CanResume: true
+DSORZ : File: empty
+DSORZ --> Transferring
+
+DSOZ : Filename: "foo.bar"
+DSOZ : Size: > 0
+DSOZ : CanResume: false
+DSOZ : File: empty
+DSOZ --> Transferring
+
+Transferring --> [*]
+Transferring --> ERROR
+
+ERROR : Something went wrong
+ERROR --> [*]
+
+@enduml

vendor/github.com/cavaliercoder/grab/transfer.go

@@ -0,0 +1,81 @@
+package grab
+
+import (
+	"context"
+	"io"
+	"sync/atomic"
+)
+
+type transfer struct {
+	n   int64 // must be 64bit aligned on 386
+	ctx context.Context
+	lim RateLimiter
+	w   io.Writer
+	r   io.Reader
+	b   []byte
+}
+
+func newTransfer(ctx context.Context, lim RateLimiter, dst io.Writer, src io.Reader, buf []byte) *transfer {
+	return &transfer{
+		ctx: ctx,
+		lim: lim,
+		w:   dst,
+		r:   src,
+		b:   buf,
+	}
+}
+
+// copy behaves similarly to io.CopyBuffer except that it checks for cancelation
+// of the given context.Context and reports progress in a thread-safe manner.
+func (c *transfer) copy() (written int64, err error) {
+	if c.b == nil {
+		c.b = make([]byte, 32*1024)
+	}
+	for {
+		select {
+		case <-c.ctx.Done():
+			err = c.ctx.Err()
+			return
+		default:
+			// keep working
+		}
+		if c.lim != nil {
+			err = c.lim.WaitN(c.ctx, len(c.b))
+			if err != nil {
+				return
+			}
+		}
+		nr, er := c.r.Read(c.b)
+		if nr > 0 {
+			nw, ew := c.w.Write(c.b[0:nr])
+			if nw > 0 {
+				written += int64(nw)
+				atomic.StoreInt64(&c.n, written)
+			}
+			if ew != nil {
+				err = ew
+				break
+			}
+			if nr != nw {
+				err = io.ErrShortWrite
+				break
+			}
+		}
+		if er != nil {
+			if er != io.EOF {
+				err = er
+			}
+			break
+		}
+	}
+	return written, err
+}
+
+// N returns the number of bytes transferred.
+func (c *transfer) N() (n int64) {
+	if c == nil {
+		return 0
+	}
+	n = atomic.LoadInt64(&c.n)
+	return
+}

vendor/github.com/cavaliercoder/grab/util.go

@@ -0,0 +1,89 @@
+package grab
+
+import (
+	"context"
+	"fmt"
+	"hash"
+	"mime"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// setLastModified sets the last modified timestamp of a local file according to
+// the Last-Modified header returned by a remote server.
+func setLastModified(resp *http.Response, filename string) error {
+	// https://tools.ietf.org/html/rfc7232#section-2.2
+	// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified
+	header := resp.Header.Get("Last-Modified")
+	if header == "" {
+		return nil
+	}
+	lastmod, err := time.Parse(http.TimeFormat, header)
+	if err != nil {
+		return nil
+	}
+	return os.Chtimes(filename, lastmod, lastmod)
+}
+
+// mkdirp creates all missing parent directories for the destination file path.
+func mkdirp(path string) error {
+	dir := filepath.Dir(path)
+	if fi, err := os.Stat(dir); err != nil {
+		if !os.IsNotExist(err) {
+			return fmt.Errorf("error checking destination directory: %v", err)
+		}
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("error creating destination directory: %v", err)
+		}
+	} else if !fi.IsDir() {
+		panic("destination path is not directory")
+	}
+	return nil
+}
+
+// guessFilename returns a filename for the given http.Response. If none can be
+// determined ErrNoFilename is returned.
+func guessFilename(resp *http.Response) (string, error) {
+	filename := resp.Request.URL.Path
+	if cd := resp.Header.Get("Content-Disposition"); cd != "" {
+		if _, params, err := mime.ParseMediaType(cd); err == nil {
+			filename = params["filename"]
+		}
+	}
+
+	// sanitize
+	if filename == "" || strings.HasSuffix(filename, "/") || strings.Contains(filename, "\x00") {
+		return "", ErrNoFilename
+	}
+
+	filename = filepath.Base(path.Clean("/" + filename))
+	if filename == "" || filename == "." || filename == "/" {
+		return "", ErrNoFilename
+	}
+
+	return filename, nil
+}
+
+// checksum returns a hash of the given file, using the given hash algorithm.
+func checksum(ctx context.Context, filename string, h hash.Hash) (b []byte, err error) {
+	var f *os.File
+	f, err = os.Open(filename)
+	if err != nil {
+		return
+	}
+	defer func() {
+		err = f.Close()
+	}()
+
+	t := newTransfer(ctx, nil, h, f, nil)
+	if _, err = t.copy(); err != nil {
+		return
+	}
+
+	b = h.Sum(nil)
+	return
+}

vendor/modules.txt

@@ -34,6 +34,8 @@ github.com/asdine/storm/index
 github.com/asdine/storm/internal
 # github.com/briandowns/spinner v1.7.0
 github.com/briandowns/spinner
+# github.com/cavaliercoder/grab v2.0.0+incompatible
+github.com/cavaliercoder/grab
 # github.com/containerd/containerd v1.2.4
 github.com/containerd/containerd/archive
 github.com/containerd/containerd/log