mirror of
https://github.com/mudler/luet.git
synced 2025-08-28 11:50:37 +00:00
update vendor/
This commit is contained in:
parent
5915688830
commit
1e426b93a1
13
vendor/github.com/cavaliercoder/grab/.travis.yml
generated
vendored
Normal file
13
vendor/github.com/cavaliercoder/grab/.travis.yml
generated
vendored
Normal file
@ -0,0 +1,13 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- 1.10.x
|
||||
- 1.9.x
|
||||
- 1.8.x
|
||||
- 1.7.x
|
||||
|
||||
script: make check
|
||||
|
||||
env:
|
||||
- GOARCH=amd64
|
||||
- GOARCH=386
|
26
vendor/github.com/cavaliercoder/grab/LICENSE
generated
vendored
Normal file
26
vendor/github.com/cavaliercoder/grab/LICENSE
generated
vendored
Normal file
@ -0,0 +1,26 @@
|
||||
Copyright (c) 2017 Ryan Armstrong. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
vendor/github.com/cavaliercoder/grab/Makefile
generated
vendored
Normal file
29
vendor/github.com/cavaliercoder/grab/Makefile
generated
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
GO = go
|
||||
GOGET = $(GO) get -u
|
||||
|
||||
all: check lint
|
||||
|
||||
check:
|
||||
cd cmd/grab && $(MAKE) -B all
|
||||
$(GO) test -cover -race ./...
|
||||
|
||||
install:
|
||||
$(GO) install -v ./...
|
||||
|
||||
clean:
|
||||
$(GO) clean -x ./...
|
||||
rm -rvf ./.test*
|
||||
|
||||
lint:
|
||||
gofmt -l -e -s . || :
|
||||
go vet . || :
|
||||
golint . || :
|
||||
gocyclo -over 15 . || :
|
||||
misspell ./* || :
|
||||
|
||||
deps:
|
||||
$(GOGET) github.com/golang/lint/golint
|
||||
$(GOGET) github.com/fzipp/gocyclo
|
||||
$(GOGET) github.com/client9/misspell/cmd/misspell
|
||||
|
||||
.PHONY: all check install clean lint deps
|
127
vendor/github.com/cavaliercoder/grab/README.md
generated
vendored
Normal file
127
vendor/github.com/cavaliercoder/grab/README.md
generated
vendored
Normal file
@ -0,0 +1,127 @@
|
||||
# grab
|
||||
|
||||
[](https://godoc.org/github.com/cavaliercoder/grab) [](https://travis-ci.org/cavaliercoder/grab) [](https://goreportcard.com/report/github.com/cavaliercoder/grab)
|
||||
|
||||
*Downloading the internet, one goroutine at a time!*
|
||||
|
||||
$ go get github.com/cavaliercoder/grab
|
||||
|
||||
Grab is a Go package for downloading files from the internet with the following
|
||||
rad features:
|
||||
|
||||
* Monitor download progress concurrently
|
||||
* Auto-resume incomplete downloads
|
||||
* Guess filename from content header or URL path
|
||||
* Safely cancel downloads using context.Context
|
||||
* Validate downloads using checksums
|
||||
* Download batches of files concurrently
|
||||
* Apply rate limiters
|
||||
|
||||
Requires Go v1.7+
|
||||
|
||||
## Example
|
||||
|
||||
The following example downloads a PDF copy of the free eBook, "An Introduction
|
||||
to Programming in Go" into the current working directory.
|
||||
|
||||
```go
|
||||
resp, err := grab.Get(".", "http://www.golang-book.com/public/pdf/gobook.pdf")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println("Download saved to", resp.Filename)
|
||||
```
|
||||
|
||||
The following, more complete example allows for more granular control and
|
||||
periodically prints the download progress until it is complete.
|
||||
|
||||
The second time you run the example, it will auto-resume the previous download
|
||||
and exit sooner.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/cavaliercoder/grab"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// create client
|
||||
client := grab.NewClient()
|
||||
req, _ := grab.NewRequest(".", "http://www.golang-book.com/public/pdf/gobook.pdf")
|
||||
|
||||
// start download
|
||||
fmt.Printf("Downloading %v...\n", req.URL())
|
||||
resp := client.Do(req)
|
||||
fmt.Printf(" %v\n", resp.HTTPResponse.Status)
|
||||
|
||||
// start UI loop
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
fmt.Printf(" transferred %v / %v bytes (%.2f%%)\n",
|
||||
resp.BytesComplete(),
|
||||
resp.Size,
|
||||
100*resp.Progress())
|
||||
|
||||
case <-resp.Done:
|
||||
// download is complete
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
|
||||
// check for errors
|
||||
if err := resp.Err(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Download failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("Download saved to ./%v \n", resp.Filename)
|
||||
|
||||
// Output:
|
||||
// Downloading http://www.golang-book.com/public/pdf/gobook.pdf...
|
||||
// 200 OK
|
||||
// transferred 42970 / 2893557 bytes (1.49%)
|
||||
// transferred 1207474 / 2893557 bytes (41.73%)
|
||||
// transferred 2758210 / 2893557 bytes (95.32%)
|
||||
// Download saved to ./gobook.pdf
|
||||
}
|
||||
```
|
||||
|
||||
## Design trade-offs
|
||||
|
||||
The primary use case for Grab is to concurrently downloading thousands of large
|
||||
files from remote file repositories where the remote files are immutable.
|
||||
Examples include operating system package repositories or ISO libraries.
|
||||
|
||||
Grab aims to provide robust, sane defaults. These are usually determined using
|
||||
the HTTP specifications, or by mimicking the behavior of common web clients like
|
||||
cURL, wget and common web browsers.
|
||||
|
||||
Grab aims to be stateless. The only state that exists is the remote files you
|
||||
wish to download and the local copy which may be completed, partially completed
|
||||
or not yet created. The advantage to this is that the local file system is not
|
||||
cluttered unnecessarily with addition state files (like a `.crdownload` file).
|
||||
The disadvantage of this approach is that grab must make assumptions about the
|
||||
local and remote state; specifically, that they have not been modified by
|
||||
another program.
|
||||
|
||||
If the local or remote file are modified outside of grab, and you download the
|
||||
file again with resuming enabled, the local file will likely become corrupted.
|
||||
In this case, you might consider making remote files immutable, or disabling
|
||||
resume.
|
||||
|
||||
Grab aims to enable best-in-class functionality for more complex features
|
||||
through extensible interfaces, rather than reimplementation. For example,
|
||||
you can provide your own Hash algorithm to compute file checksums, or your
|
||||
own rate limiter implementation (with all the associated trade-offs) to rate
|
||||
limit downloads.
|
506
vendor/github.com/cavaliercoder/grab/client.go
generated
vendored
Normal file
506
vendor/github.com/cavaliercoder/grab/client.go
generated
vendored
Normal file
@ -0,0 +1,506 @@
|
||||
package grab
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A Client is a file download client.
|
||||
//
|
||||
// Clients are safe for concurrent use by multiple goroutines.
|
||||
type Client struct {
|
||||
// HTTPClient specifies the http.Client which will be used for communicating
|
||||
// with the remote server during the file transfer.
|
||||
HTTPClient *http.Client
|
||||
|
||||
// UserAgent specifies the User-Agent string which will be set in the
|
||||
// headers of all requests made by this client.
|
||||
//
|
||||
// The user agent string may be overridden in the headers of each request.
|
||||
UserAgent string
|
||||
|
||||
// BufferSize specifies the size in bytes of the buffer that is used for
|
||||
// transferring all requested files. Larger buffers may result in faster
|
||||
// throughput but will use more memory and result in less frequent updates
|
||||
// to the transfer progress statistics. The BufferSize of each request can
|
||||
// be overridden on each Request object. Default: 32KB.
|
||||
BufferSize int
|
||||
}
|
||||
|
||||
// NewClient returns a new file download Client, using default configuration.
|
||||
func NewClient() *Client {
|
||||
return &Client{
|
||||
UserAgent: "grab",
|
||||
HTTPClient: &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultClient is the default client and is used by all Get convenience
|
||||
// functions.
|
||||
var DefaultClient = NewClient()
|
||||
|
||||
// Do sends a file transfer request and returns a file transfer response,
|
||||
// following policy (e.g. redirects, cookies, auth) as configured on the
|
||||
// client's HTTPClient.
|
||||
//
|
||||
// Like http.Get, Do blocks while the transfer is initiated, but returns as soon
|
||||
// as the transfer has started transferring in a background goroutine, or if it
|
||||
// failed early.
|
||||
//
|
||||
// An error is returned via Response.Err if caused by client policy (such as
|
||||
// CheckRedirect), or if there was an HTTP protocol or IO error. Response.Err
|
||||
// will block the caller until the transfer is completed, successfully or
|
||||
// otherwise.
|
||||
func (c *Client) Do(req *Request) *Response {
|
||||
// cancel will be called on all code-paths via closeResponse
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
resp := &Response{
|
||||
Request: req,
|
||||
Start: time.Now(),
|
||||
Done: make(chan struct{}, 0),
|
||||
Filename: req.Filename,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
bufferSize: req.BufferSize,
|
||||
}
|
||||
if resp.bufferSize == 0 {
|
||||
// default to Client.BufferSize
|
||||
resp.bufferSize = c.BufferSize
|
||||
}
|
||||
|
||||
// Run state-machine while caller is blocked to initialize the file transfer.
|
||||
// Must never transition to the copyFile state - this happens next in another
|
||||
// goroutine.
|
||||
c.run(resp, c.statFileInfo)
|
||||
|
||||
// Run copyFile in a new goroutine. copyFile will no-op if the transfer is
|
||||
// already complete or failed.
|
||||
go c.run(resp, c.copyFile)
|
||||
return resp
|
||||
}
|
||||
|
||||
// DoChannel executes all requests sent through the given Request channel, one
|
||||
// at a time, until it is closed by another goroutine. The caller is blocked
|
||||
// until the Request channel is closed and all transfers have completed. All
|
||||
// responses are sent through the given Response channel as soon as they are
|
||||
// received from the remote servers and can be used to track the progress of
|
||||
// each download.
|
||||
//
|
||||
// Slow Response receivers will cause a worker to block and therefore delay the
|
||||
// start of the transfer for an already initiated connection - potentially
|
||||
// causing a server timeout. It is the caller's responsibility to ensure a
|
||||
// sufficient buffer size is used for the Response channel to prevent this.
|
||||
//
|
||||
// If an error occurs during any of the file transfers it will be accessible via
|
||||
// the associated Response.Err function.
|
||||
func (c *Client) DoChannel(reqch <-chan *Request, respch chan<- *Response) {
|
||||
// TODO: enable cancelling of batch jobs
|
||||
for req := range reqch {
|
||||
resp := c.Do(req)
|
||||
respch <- resp
|
||||
<-resp.Done
|
||||
}
|
||||
}
|
||||
|
||||
// DoBatch executes all the given requests using the given number of concurrent
|
||||
// workers. Control is passed back to the caller as soon as the workers are
|
||||
// initiated.
|
||||
//
|
||||
// If the requested number of workers is less than one, a worker will be created
|
||||
// for every request. I.e. all requests will be executed concurrently.
|
||||
//
|
||||
// If an error occurs during any of the file transfers it will be accessible via
|
||||
// call to the associated Response.Err.
|
||||
//
|
||||
// The returned Response channel is closed only after all of the given Requests
|
||||
// have completed, successfully or otherwise.
|
||||
func (c *Client) DoBatch(workers int, requests ...*Request) <-chan *Response {
|
||||
if workers < 1 {
|
||||
workers = len(requests)
|
||||
}
|
||||
reqch := make(chan *Request, len(requests))
|
||||
respch := make(chan *Response, len(requests))
|
||||
wg := sync.WaitGroup{}
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
c.DoChannel(reqch, respch)
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
// queue requests
|
||||
go func() {
|
||||
for _, req := range requests {
|
||||
reqch <- req
|
||||
}
|
||||
close(reqch)
|
||||
wg.Wait()
|
||||
close(respch)
|
||||
}()
|
||||
return respch
|
||||
}
|
||||
|
||||
// An stateFunc is an action that mutates the state of a Response and returns
|
||||
// the next stateFunc to be called.
|
||||
type stateFunc func(*Response) stateFunc
|
||||
|
||||
// run calls the given stateFunc function and all subsequent returned stateFuncs
|
||||
// until a stateFunc returns nil or the Response.ctx is canceled. Each stateFunc
|
||||
// should mutate the state of the given Response until it has completed
|
||||
// downloading or failed.
|
||||
func (c *Client) run(resp *Response, f stateFunc) {
|
||||
for {
|
||||
select {
|
||||
case <-resp.ctx.Done():
|
||||
if resp.IsComplete() {
|
||||
return
|
||||
}
|
||||
resp.err = resp.ctx.Err()
|
||||
f = c.closeResponse
|
||||
|
||||
default:
|
||||
// keep working
|
||||
}
|
||||
if f = f(resp); f == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// statFileInfo retrieves FileInfo for any local file matching
|
||||
// Response.Filename.
|
||||
//
|
||||
// If the file does not exist, is a directory, or its name is unknown the next
|
||||
// stateFunc is headRequest.
|
||||
//
|
||||
// If the file exists, Response.fi is set and the next stateFunc is
|
||||
// validateLocal.
|
||||
//
|
||||
// If an error occurs, the next stateFunc is closeResponse.
|
||||
func (c *Client) statFileInfo(resp *Response) stateFunc {
|
||||
if resp.Filename == "" {
|
||||
return c.headRequest
|
||||
}
|
||||
fi, err := os.Stat(resp.Filename)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return c.headRequest
|
||||
}
|
||||
resp.err = err
|
||||
return c.closeResponse
|
||||
}
|
||||
if fi.IsDir() {
|
||||
resp.Filename = ""
|
||||
return c.headRequest
|
||||
}
|
||||
resp.fi = fi
|
||||
return c.validateLocal
|
||||
}
|
||||
|
||||
// validateLocal compares a local copy of the downloaded file to the remote
|
||||
// file.
|
||||
//
|
||||
// An error is returned if the local file is larger than the remote file, or
|
||||
// Request.SkipExisting is true.
|
||||
//
|
||||
// If the existing file matches the length of the remote file, the next
|
||||
// stateFunc is checksumFile.
|
||||
//
|
||||
// If the local file is smaller than the remote file and the remote server is
|
||||
// known to support ranged requests, the next stateFunc is getRequest.
|
||||
func (c *Client) validateLocal(resp *Response) stateFunc {
|
||||
if resp.Request.SkipExisting {
|
||||
resp.err = ErrFileExists
|
||||
return c.closeResponse
|
||||
}
|
||||
|
||||
// determine expected file size
|
||||
size := resp.Request.Size
|
||||
if size == 0 && resp.HTTPResponse != nil {
|
||||
size = resp.HTTPResponse.ContentLength
|
||||
}
|
||||
if size == 0 {
|
||||
return c.headRequest
|
||||
}
|
||||
|
||||
if size == resp.fi.Size() {
|
||||
resp.DidResume = true
|
||||
resp.bytesResumed = resp.fi.Size()
|
||||
return c.checksumFile
|
||||
}
|
||||
|
||||
if resp.Request.NoResume {
|
||||
return c.getRequest
|
||||
}
|
||||
|
||||
if size < resp.fi.Size() {
|
||||
resp.err = ErrBadLength
|
||||
return c.closeResponse
|
||||
}
|
||||
|
||||
if resp.CanResume {
|
||||
resp.Request.HTTPRequest.Header.Set(
|
||||
"Range",
|
||||
fmt.Sprintf("bytes=%d-", resp.fi.Size()))
|
||||
resp.DidResume = true
|
||||
resp.bytesResumed = resp.fi.Size()
|
||||
return c.getRequest
|
||||
}
|
||||
return c.headRequest
|
||||
}
|
||||
|
||||
func (c *Client) checksumFile(resp *Response) stateFunc {
|
||||
if resp.Request.hash == nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
if resp.Filename == "" {
|
||||
panic("filename not set")
|
||||
}
|
||||
req := resp.Request
|
||||
|
||||
// compare checksum
|
||||
var sum []byte
|
||||
sum, resp.err = checksum(req.Context(), resp.Filename, req.hash)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
if !bytes.Equal(sum, req.checksum) {
|
||||
resp.err = ErrBadChecksum
|
||||
if req.deleteOnError {
|
||||
if err := os.Remove(resp.Filename); err != nil {
|
||||
// err should be os.PathError and include file path
|
||||
resp.err = fmt.Errorf(
|
||||
"cannot remove downloaded file with checksum mismatch: %v",
|
||||
err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return c.closeResponse
|
||||
}
|
||||
|
||||
// doHTTPRequest sends a HTTP Request and returns the response
|
||||
func (c *Client) doHTTPRequest(req *http.Request) (*http.Response, error) {
|
||||
if c.UserAgent != "" && req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", c.UserAgent)
|
||||
}
|
||||
return c.HTTPClient.Do(req)
|
||||
}
|
||||
|
||||
func (c *Client) headRequest(resp *Response) stateFunc {
|
||||
if resp.optionsKnown {
|
||||
return c.getRequest
|
||||
}
|
||||
resp.optionsKnown = true
|
||||
|
||||
if resp.Request.NoResume {
|
||||
return c.getRequest
|
||||
}
|
||||
|
||||
if resp.Filename != "" && resp.fi == nil {
|
||||
// destination path is already known and does not exist
|
||||
return c.getRequest
|
||||
}
|
||||
|
||||
hreq := new(http.Request)
|
||||
*hreq = *resp.Request.HTTPRequest
|
||||
hreq.Method = "HEAD"
|
||||
|
||||
resp.HTTPResponse, resp.err = c.doHTTPRequest(hreq)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
resp.HTTPResponse.Body.Close()
|
||||
|
||||
if resp.HTTPResponse.StatusCode != http.StatusOK {
|
||||
return c.getRequest
|
||||
}
|
||||
|
||||
return c.readResponse
|
||||
}
|
||||
|
||||
func (c *Client) getRequest(resp *Response) stateFunc {
|
||||
resp.HTTPResponse, resp.err = c.doHTTPRequest(resp.Request.HTTPRequest)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
|
||||
// check status code
|
||||
if !resp.Request.IgnoreBadStatusCodes {
|
||||
if resp.HTTPResponse.StatusCode < 200 || resp.HTTPResponse.StatusCode > 299 {
|
||||
resp.err = StatusCodeError(resp.HTTPResponse.StatusCode)
|
||||
return c.closeResponse
|
||||
}
|
||||
}
|
||||
|
||||
return c.readResponse
|
||||
}
|
||||
|
||||
func (c *Client) readResponse(resp *Response) stateFunc {
|
||||
if resp.HTTPResponse == nil {
|
||||
panic("Response.HTTPResponse is not ready")
|
||||
}
|
||||
|
||||
// check expected size
|
||||
resp.Size = resp.bytesResumed + resp.HTTPResponse.ContentLength
|
||||
if resp.HTTPResponse.ContentLength > 0 && resp.Request.Size > 0 {
|
||||
if resp.Request.Size != resp.Size {
|
||||
resp.err = ErrBadLength
|
||||
return c.closeResponse
|
||||
}
|
||||
}
|
||||
|
||||
// check filename
|
||||
if resp.Filename == "" {
|
||||
filename, err := guessFilename(resp.HTTPResponse)
|
||||
if err != nil {
|
||||
resp.err = err
|
||||
return c.closeResponse
|
||||
}
|
||||
// Request.Filename will be empty or a directory
|
||||
resp.Filename = filepath.Join(resp.Request.Filename, filename)
|
||||
}
|
||||
|
||||
if resp.requestMethod() == "HEAD" {
|
||||
if resp.HTTPResponse.Header.Get("Accept-Ranges") == "bytes" {
|
||||
resp.CanResume = true
|
||||
}
|
||||
return c.statFileInfo
|
||||
}
|
||||
return c.openWriter
|
||||
}
|
||||
|
||||
// openWriter opens the destination file for writing and seeks to the location
|
||||
// from whence the file transfer will resume.
|
||||
//
|
||||
// Requires that Response.Filename and resp.DidResume are already be set.
|
||||
func (c *Client) openWriter(resp *Response) stateFunc {
|
||||
if !resp.Request.NoCreateDirectories {
|
||||
resp.err = mkdirp(resp.Filename)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
}
|
||||
|
||||
// compute write flags
|
||||
flag := os.O_CREATE | os.O_WRONLY
|
||||
if resp.fi != nil {
|
||||
if resp.DidResume {
|
||||
flag = os.O_APPEND | os.O_WRONLY
|
||||
} else {
|
||||
flag = os.O_TRUNC | os.O_WRONLY
|
||||
}
|
||||
}
|
||||
|
||||
// open file
|
||||
f, err := os.OpenFile(resp.Filename, flag, 0644)
|
||||
if err != nil {
|
||||
resp.err = err
|
||||
return c.closeResponse
|
||||
}
|
||||
resp.writer = f
|
||||
|
||||
// seek to start or end
|
||||
whence := os.SEEK_SET
|
||||
if resp.bytesResumed > 0 {
|
||||
whence = os.SEEK_END
|
||||
}
|
||||
_, resp.err = f.Seek(0, whence)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
|
||||
// init transfer
|
||||
if resp.bufferSize < 1 {
|
||||
resp.bufferSize = 32 * 1024
|
||||
}
|
||||
b := make([]byte, resp.bufferSize)
|
||||
resp.transfer = newTransfer(
|
||||
resp.Request.Context(),
|
||||
resp.Request.RateLimiter,
|
||||
resp.writer,
|
||||
resp.HTTPResponse.Body,
|
||||
b)
|
||||
|
||||
// next step is copyFile, but this will be called later in another goroutine
|
||||
return nil
|
||||
}
|
||||
|
||||
// copy transfers content for a HTTP connection established via Client.do()
|
||||
func (c *Client) copyFile(resp *Response) stateFunc {
|
||||
if resp.IsComplete() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// run BeforeCopy hook
|
||||
if f := resp.Request.BeforeCopy; f != nil {
|
||||
resp.err = f(resp)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
}
|
||||
|
||||
if resp.transfer == nil {
|
||||
panic("developer error: Response.transfer is not initialized")
|
||||
}
|
||||
go resp.watchBps()
|
||||
_, resp.err = resp.transfer.copy()
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
closeWriter(resp)
|
||||
|
||||
// set timestamp
|
||||
if !resp.Request.IgnoreRemoteTime {
|
||||
resp.err = setLastModified(resp.HTTPResponse, resp.Filename)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
}
|
||||
|
||||
// run AfterCopy hook
|
||||
if f := resp.Request.AfterCopy; f != nil {
|
||||
resp.err = f(resp)
|
||||
if resp.err != nil {
|
||||
return c.closeResponse
|
||||
}
|
||||
}
|
||||
|
||||
return c.checksumFile
|
||||
}
|
||||
|
||||
func closeWriter(resp *Response) {
|
||||
if resp.writer != nil {
|
||||
resp.writer.Close()
|
||||
resp.writer = nil
|
||||
}
|
||||
}
|
||||
|
||||
// close finalizes the Response
|
||||
func (c *Client) closeResponse(resp *Response) stateFunc {
|
||||
if resp.IsComplete() {
|
||||
panic("response already closed")
|
||||
}
|
||||
|
||||
resp.fi = nil
|
||||
closeWriter(resp)
|
||||
resp.closeResponseBody()
|
||||
|
||||
resp.End = time.Now()
|
||||
close(resp.Done)
|
||||
if resp.cancel != nil {
|
||||
resp.cancel()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
63
vendor/github.com/cavaliercoder/grab/doc.go
generated
vendored
Normal file
63
vendor/github.com/cavaliercoder/grab/doc.go
generated
vendored
Normal file
@ -0,0 +1,63 @@
|
||||
/*
|
||||
Package grab provides a HTTP download manager implementation.
|
||||
|
||||
Get is the most simple way to download a file:
|
||||
|
||||
resp, err := grab.Get("/tmp", "http://example.com/example.zip")
|
||||
// ...
|
||||
|
||||
Get will download the given URL and save it to the given destination directory.
|
||||
The destination filename will be determined automatically by grab using
|
||||
Content-Disposition headers returned by the remote server, or by inspecting the
|
||||
requested URL path.
|
||||
|
||||
An empty destination string or "." means the transfer will be stored in the
|
||||
current working directory.
|
||||
|
||||
If a destination file already exists, grab will assume it is a complete or
|
||||
partially complete download of the requested file. If the remote server supports
|
||||
resuming interrupted downloads, grab will resume downloading from the end of the
|
||||
partial file. If the server does not support resumed downloads, the file will be
|
||||
retransferred in its entirety. If the file is already complete, grab will return
|
||||
successfully.
|
||||
|
||||
For control over the HTTP client, destination path, auto-resume, checksum
|
||||
validation and other settings, create a Client:
|
||||
|
||||
client := grab.NewClient()
|
||||
client.HTTPClient.Transport.DisableCompression = true
|
||||
|
||||
req, err := grab.NewRequest("/tmp", "http://example.com/example.zip")
|
||||
// ...
|
||||
req.NoResume = true
|
||||
req.HTTPRequest.Header.Set("Authorization", "Basic YWxhZGRpbjpvcGVuc2VzYW1l")
|
||||
|
||||
resp := client.Do(req)
|
||||
// ...
|
||||
|
||||
You can monitor the progress of downloads while they are transferring:
|
||||
|
||||
client := grab.NewClient()
|
||||
req, err := grab.NewRequest("", "http://example.com/example.zip")
|
||||
// ...
|
||||
resp := client.Do(req)
|
||||
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
fmt.Printf("%.02f%% complete\n", resp.Progress())
|
||||
|
||||
case <-resp.Done:
|
||||
if err := resp.Err(); err != nil {
|
||||
// ...
|
||||
}
|
||||
|
||||
// ...
|
||||
return
|
||||
}
|
||||
}
|
||||
*/
|
||||
package grab
|
42
vendor/github.com/cavaliercoder/grab/error.go
generated
vendored
Normal file
42
vendor/github.com/cavaliercoder/grab/error.go
generated
vendored
Normal file
@ -0,0 +1,42 @@
|
||||
package grab
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrBadLength indicates that the server response or an existing file does
|
||||
// not match the expected content length.
|
||||
ErrBadLength = errors.New("bad content length")
|
||||
|
||||
// ErrBadChecksum indicates that a downloaded file failed to pass checksum
|
||||
// validation.
|
||||
ErrBadChecksum = errors.New("checksum mismatch")
|
||||
|
||||
// ErrNoFilename indicates that a reasonable filename could not be
|
||||
// automatically determined using the URL or response headers from a server.
|
||||
ErrNoFilename = errors.New("no filename could be determined")
|
||||
|
||||
// ErrNoTimestamp indicates that a timestamp could not be automatically
|
||||
// determined using the response headers from the remote server.
|
||||
ErrNoTimestamp = errors.New("no timestamp could be determined for the remote file")
|
||||
|
||||
// ErrFileExists indicates that the destination path already exists.
|
||||
ErrFileExists = errors.New("file exists")
|
||||
)
|
||||
|
||||
// StatusCodeError indicates that the server response had a status code that
|
||||
// was not in the 200-299 range (after following any redirects).
|
||||
type StatusCodeError int
|
||||
|
||||
func (err StatusCodeError) Error() string {
|
||||
return fmt.Sprintf("server returned %d %s", err, http.StatusText(int(err)))
|
||||
}
|
||||
|
||||
// IsStatusCodeError returns true if the given error is of type StatusCodeError.
|
||||
func IsStatusCodeError(err error) bool {
|
||||
_, ok := err.(StatusCodeError)
|
||||
return ok
|
||||
}
|
64
vendor/github.com/cavaliercoder/grab/grab.go
generated
vendored
Normal file
64
vendor/github.com/cavaliercoder/grab/grab.go
generated
vendored
Normal file
@ -0,0 +1,64 @@
|
||||
package grab
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Get sends a HTTP request and downloads the content of the requested URL to
|
||||
// the given destination file path. The caller is blocked until the download is
|
||||
// completed, successfully or otherwise.
|
||||
//
|
||||
// An error is returned if caused by client policy (such as CheckRedirect), or
|
||||
// if there was an HTTP protocol or IO error.
|
||||
//
|
||||
// For non-blocking calls or control over HTTP client headers, redirect policy,
|
||||
// and other settings, create a Client instead.
|
||||
func Get(dst, urlStr string) (*Response, error) {
|
||||
req, err := NewRequest(dst, urlStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp := DefaultClient.Do(req)
|
||||
return resp, resp.Err()
|
||||
}
|
||||
|
||||
// GetBatch sends multiple HTTP requests and downloads the content of the
|
||||
// requested URLs to the given destination directory using the given number of
|
||||
// concurrent worker goroutines.
|
||||
//
|
||||
// The Response for each requested URL is sent through the returned Response
|
||||
// channel, as soon as a worker receives a response from the remote server. The
|
||||
// Response can then be used to track the progress of the download while it is
|
||||
// in progress.
|
||||
//
|
||||
// The returned Response channel will be closed by Grab, only once all downloads
|
||||
// have completed or failed.
|
||||
//
|
||||
// If an error occurs during any download, it will be available via call to the
|
||||
// associated Response.Err.
|
||||
//
|
||||
// For control over HTTP client headers, redirect policy, and other settings,
|
||||
// create a Client instead.
|
||||
func GetBatch(workers int, dst string, urlStrs ...string) (<-chan *Response, error) {
|
||||
fi, err := os.Stat(dst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return nil, fmt.Errorf("destination is not a directory")
|
||||
}
|
||||
|
||||
reqs := make([]*Request, len(urlStrs))
|
||||
for i := 0; i < len(urlStrs); i++ {
|
||||
req, err := NewRequest(dst, urlStrs[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqs[i] = req
|
||||
}
|
||||
|
||||
ch := DefaultClient.DoBatch(workers, reqs...)
|
||||
return ch, nil
|
||||
}
|
12
vendor/github.com/cavaliercoder/grab/rate_limiter.go
generated
vendored
Normal file
12
vendor/github.com/cavaliercoder/grab/rate_limiter.go
generated
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
package grab
|
||||
|
||||
import "context"
|
||||
|
||||
// RateLimiter is an interface that must be satisfied by any third-party rate
|
||||
// limiters that may be used to limit download transfer speeds.
|
||||
//
|
||||
// A recommended token bucket implementation can be found at
|
||||
// https://godoc.org/golang.org/x/time/rate#Limiter.
|
||||
type RateLimiter interface {
|
||||
WaitN(ctx context.Context, n int) (err error)
|
||||
}
|
172
vendor/github.com/cavaliercoder/grab/request.go
generated
vendored
Normal file
172
vendor/github.com/cavaliercoder/grab/request.go
generated
vendored
Normal file
@ -0,0 +1,172 @@
|
||||
package grab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"hash"
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// A Hook is a user provided callback function that can be called by grab at
|
||||
// various stages of a requests lifecycle. If a hook returns an error, the
|
||||
// associated request is canceled and the same error is returned on the Response
|
||||
// object.
|
||||
//
|
||||
// Hook functions are called synchronously and should never block unnecessarily.
|
||||
// Response methods that block until a download is complete, such as
|
||||
// Response.Err, Response.Cancel or Response.Wait will deadlock. To cancel a
|
||||
// download from a callback, simply return a non-nil error.
|
||||
type Hook func(*Response) error
|
||||
|
||||
// A Request represents an HTTP file transfer request to be sent by a Client.
|
||||
type Request struct {
|
||||
// Label is an arbitrary string which may used to label a Request with a
|
||||
// user friendly name.
|
||||
Label string
|
||||
|
||||
// Tag is an arbitrary interface which may be used to relate a Request to
|
||||
// other data.
|
||||
Tag interface{}
|
||||
|
||||
// HTTPRequest specifies the http.Request to be sent to the remote server to
|
||||
// initiate a file transfer. It includes request configuration such as URL,
|
||||
// protocol version, HTTP method, request headers and authentication.
|
||||
HTTPRequest *http.Request
|
||||
|
||||
// Filename specifies the path where the file transfer will be stored in
|
||||
// local storage. If Filename is empty or a directory, the true Filename will
|
||||
// be resolved using Content-Disposition headers or the request URL.
|
||||
//
|
||||
// An empty string means the transfer will be stored in the current working
|
||||
// directory.
|
||||
Filename string
|
||||
|
||||
// SkipExisting specifies that ErrFileExists should be returned if the
|
||||
// destination path already exists. The existing file will not be checked for
|
||||
// completeness.
|
||||
SkipExisting bool
|
||||
|
||||
// NoResume specifies that a partially completed download will be restarted
|
||||
// without attempting to resume any existing file. If the download is already
|
||||
// completed in full, it will not be restarted.
|
||||
NoResume bool
|
||||
|
||||
// NoCreateDirectories specifies that any missing directories in the given
|
||||
// Filename path should not be created automatically, if they do not already
|
||||
// exist.
|
||||
NoCreateDirectories bool
|
||||
|
||||
// IgnoreBadStatusCodes specifies that grab should accept any status code in
|
||||
// the response from the remote server. Otherwise, grab expects the response
|
||||
// status code to be within the 2XX range (after following redirects).
|
||||
IgnoreBadStatusCodes bool
|
||||
|
||||
// IgnoreRemoteTime specifies that grab should not attempt to set the
|
||||
// timestamp of the local file to match the remote file.
|
||||
IgnoreRemoteTime bool
|
||||
|
||||
// Size specifies the expected size of the file transfer if known. If the
|
||||
// server response size does not match, the transfer is cancelled and
|
||||
// ErrBadLength returned.
|
||||
Size int64
|
||||
|
||||
// BufferSize specifies the size in bytes of the buffer that is used for
|
||||
// transferring the requested file. Larger buffers may result in faster
|
||||
// throughput but will use more memory and result in less frequent updates
|
||||
// to the transfer progress statistics. If a RateLimiter is configured,
|
||||
// BufferSize should be much lower than the rate limit. Default: 32KB.
|
||||
BufferSize int
|
||||
|
||||
// RateLimiter allows the transfer rate of a download to be limited. The given
|
||||
// Request.BufferSize determines how frequently the RateLimiter will be
|
||||
// polled.
|
||||
RateLimiter RateLimiter
|
||||
|
||||
// BeforeCopy is a user provided callback that is called immediately before
|
||||
// a request starts downloading. If BeforeCopy returns an error, the request
|
||||
// is cancelled and the same error is returned on the Response object.
|
||||
BeforeCopy Hook
|
||||
|
||||
// AfterCopy is a user provided callback that is called immediately after a
|
||||
// request has finished downloading, before checksum validation and closure.
|
||||
// This hook is only called if the transfer was successful. If AfterCopy
|
||||
// returns an error, the request is canceled and the same error is returned on
|
||||
// the Response object.
|
||||
AfterCopy Hook
|
||||
|
||||
// hash, checksum and deleteOnError - set via SetChecksum.
|
||||
hash hash.Hash
|
||||
checksum []byte
|
||||
deleteOnError bool
|
||||
|
||||
// Context for cancellation and timeout - set via WithContext
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// NewRequest returns a new file transfer Request suitable for use with
|
||||
// Client.Do.
|
||||
func NewRequest(dst, urlStr string) (*Request, error) {
|
||||
if dst == "" {
|
||||
dst = "."
|
||||
}
|
||||
req, err := http.NewRequest("GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Request{
|
||||
HTTPRequest: req,
|
||||
Filename: dst,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Context returns the request's context. To change the context, use
|
||||
// WithContext.
|
||||
//
|
||||
// The returned context is always non-nil; it defaults to the background
|
||||
// context.
|
||||
//
|
||||
// The context controls cancelation.
|
||||
func (r *Request) Context() context.Context {
|
||||
if r.ctx != nil {
|
||||
return r.ctx
|
||||
}
|
||||
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
// WithContext returns a shallow copy of r with its context changed
|
||||
// to ctx. The provided ctx must be non-nil.
|
||||
func (r *Request) WithContext(ctx context.Context) *Request {
|
||||
if ctx == nil {
|
||||
panic("nil context")
|
||||
}
|
||||
r2 := new(Request)
|
||||
*r2 = *r
|
||||
r2.ctx = ctx
|
||||
r2.HTTPRequest = r2.HTTPRequest.WithContext(ctx)
|
||||
return r2
|
||||
}
|
||||
|
||||
// URL returns the URL to be downloaded.
|
||||
func (r *Request) URL() *url.URL {
|
||||
return r.HTTPRequest.URL
|
||||
}
|
||||
|
||||
// SetChecksum sets the desired hashing algorithm and checksum value to validate
|
||||
// a downloaded file. Once the download is complete, the given hashing algorithm
|
||||
// will be used to compute the actual checksum of the downloaded file. If the
|
||||
// checksums do not match, an error will be returned by the associated
|
||||
// Response.Err method.
|
||||
//
|
||||
// If deleteOnError is true, the downloaded file will be deleted automatically
|
||||
// if it fails checksum validation.
|
||||
//
|
||||
// To prevent corruption of the computed checksum, the given hash must not be
|
||||
// used by any other request or goroutines.
|
||||
//
|
||||
// To disable checksum validation, call SetChecksum with a nil hash.
|
||||
func (r *Request) SetChecksum(h hash.Hash, sum []byte, deleteOnError bool) {
|
||||
r.hash = h
|
||||
r.checksum = sum
|
||||
r.deleteOnError = deleteOnError
|
||||
}
|
224
vendor/github.com/cavaliercoder/grab/response.go
generated
vendored
Normal file
224
vendor/github.com/cavaliercoder/grab/response.go
generated
vendored
Normal file
@ -0,0 +1,224 @@
|
||||
package grab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Response represents the response to a completed or in-progress download
|
||||
// request.
|
||||
//
|
||||
// A response may be returned as soon a HTTP response is received from a remote
|
||||
// server, but before the body content has started transferring.
|
||||
//
|
||||
// All Response method calls are thread-safe.
|
||||
type Response struct {
|
||||
// The Request that was submitted to obtain this Response.
|
||||
Request *Request
|
||||
|
||||
// HTTPResponse represents the HTTP response received from an HTTP request.
|
||||
//
|
||||
// The response Body should not be used as it will be consumed and closed by
|
||||
// grab.
|
||||
HTTPResponse *http.Response
|
||||
|
||||
// Filename specifies the path where the file transfer is stored in local
|
||||
// storage.
|
||||
Filename string
|
||||
|
||||
// Size specifies the total expected size of the file transfer.
|
||||
Size int64
|
||||
|
||||
// Start specifies the time at which the file transfer started.
|
||||
Start time.Time
|
||||
|
||||
// End specifies the time at which the file transfer completed.
|
||||
//
|
||||
// This will return zero until the transfer has completed.
|
||||
End time.Time
|
||||
|
||||
// CanResume specifies that the remote server advertised that it can resume
|
||||
// previous downloads, as the 'Accept-Ranges: bytes' header is set.
|
||||
CanResume bool
|
||||
|
||||
// DidResume specifies that the file transfer resumed a previously incomplete
|
||||
// transfer.
|
||||
DidResume bool
|
||||
|
||||
// Done is closed once the transfer is finalized, either successfully or with
|
||||
// errors. Errors are available via Response.Err
|
||||
Done chan struct{}
|
||||
|
||||
// ctx is a Context that controls cancelation of an inprogress transfer
|
||||
ctx context.Context
|
||||
|
||||
// cancel is a cancel func that can be used to cancel the context of this
|
||||
// Response.
|
||||
cancel context.CancelFunc
|
||||
|
||||
// fi is the FileInfo for the destination file if it already existed before
|
||||
// transfer started.
|
||||
fi os.FileInfo
|
||||
|
||||
// optionsKnown indicates that a HEAD request has been completed and the
|
||||
// capabilities of the remote server are known.
|
||||
optionsKnown bool
|
||||
|
||||
// writer is the file handle used to write the downloaded file to local
|
||||
// storage
|
||||
writer io.WriteCloser
|
||||
|
||||
// bytesCompleted specifies the number of bytes which were already
|
||||
// transferred before this transfer began.
|
||||
bytesResumed int64
|
||||
|
||||
// transfer is responsible for copying data from the remote server to a local
|
||||
// file, tracking progress and allowing for cancelation.
|
||||
transfer *transfer
|
||||
|
||||
// bytesPerSecond specifies the number of bytes that have been transferred in
|
||||
// the last 1-second window.
|
||||
bytesPerSecond float64
|
||||
bytesPerSecondMu sync.Mutex
|
||||
|
||||
// bufferSize specifies the size in bytes of the transfer buffer.
|
||||
bufferSize int
|
||||
|
||||
// Error contains any error that may have occurred during the file transfer.
|
||||
// This should not be read until IsComplete returns true.
|
||||
err error
|
||||
}
|
||||
|
||||
// IsComplete returns true if the download has completed. If an error occurred
|
||||
// during the download, it can be returned via Err.
|
||||
func (c *Response) IsComplete() bool {
|
||||
select {
|
||||
case <-c.Done:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Cancel cancels the file transfer by canceling the underlying Context for
|
||||
// this Response. Cancel blocks until the transfer is closed and returns any
|
||||
// error - typically context.Canceled.
|
||||
func (c *Response) Cancel() error {
|
||||
c.cancel()
|
||||
return c.Err()
|
||||
}
|
||||
|
||||
// Wait blocks until the download is completed.
|
||||
func (c *Response) Wait() {
|
||||
<-c.Done
|
||||
}
|
||||
|
||||
// Err blocks the calling goroutine until the underlying file transfer is
|
||||
// completed and returns any error that may have occurred. If the download is
|
||||
// already completed, Err returns immediately.
|
||||
func (c *Response) Err() error {
|
||||
<-c.Done
|
||||
return c.err
|
||||
}
|
||||
|
||||
// BytesComplete returns the total number of bytes which have been copied to
|
||||
// the destination, including any bytes that were resumed from a previous
|
||||
// download.
|
||||
func (c *Response) BytesComplete() int64 {
|
||||
return c.bytesResumed + c.transfer.N()
|
||||
}
|
||||
|
||||
// BytesPerSecond returns the number of bytes transferred in the last second. If
|
||||
// the download is already complete, the average bytes/sec for the life of the
|
||||
// download is returned.
|
||||
func (c *Response) BytesPerSecond() float64 {
|
||||
if c.IsComplete() {
|
||||
return float64(c.transfer.N()) / c.Duration().Seconds()
|
||||
}
|
||||
c.bytesPerSecondMu.Lock()
|
||||
defer c.bytesPerSecondMu.Unlock()
|
||||
return c.bytesPerSecond
|
||||
}
|
||||
|
||||
// Progress returns the ratio of total bytes that have been downloaded. Multiply
|
||||
// the returned value by 100 to return the percentage completed.
|
||||
func (c *Response) Progress() float64 {
|
||||
if c.Size == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(c.BytesComplete()) / float64(c.Size)
|
||||
}
|
||||
|
||||
// Duration returns the duration of a file transfer. If the transfer is in
|
||||
// process, the duration will be between now and the start of the transfer. If
|
||||
// the transfer is complete, the duration will be between the start and end of
|
||||
// the completed transfer process.
|
||||
func (c *Response) Duration() time.Duration {
|
||||
if c.IsComplete() {
|
||||
return c.End.Sub(c.Start)
|
||||
}
|
||||
|
||||
return time.Now().Sub(c.Start)
|
||||
}
|
||||
|
||||
// ETA returns the estimated time at which the the download will complete, given
|
||||
// the current BytesPerSecond. If the transfer has already completed, the actual
|
||||
// end time will be returned.
|
||||
func (c *Response) ETA() time.Time {
|
||||
if c.IsComplete() {
|
||||
return c.End
|
||||
}
|
||||
bt := c.BytesComplete()
|
||||
bps := c.BytesPerSecond()
|
||||
if bps == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
secs := float64(c.Size-bt) / bps
|
||||
return time.Now().Add(time.Duration(secs) * time.Second)
|
||||
}
|
||||
|
||||
// watchBps watches the progress of a transfer and maintains statistics.
|
||||
func (c *Response) watchBps() {
|
||||
var prev int64
|
||||
then := c.Start
|
||||
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-c.Done:
|
||||
return
|
||||
|
||||
case now := <-t.C:
|
||||
d := now.Sub(then)
|
||||
then = now
|
||||
|
||||
cur := c.transfer.N()
|
||||
bs := cur - prev
|
||||
prev = cur
|
||||
|
||||
c.bytesPerSecondMu.Lock()
|
||||
c.bytesPerSecond = float64(bs) / d.Seconds()
|
||||
c.bytesPerSecondMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Response) requestMethod() string {
|
||||
if c == nil || c.HTTPResponse == nil || c.HTTPResponse.Request == nil {
|
||||
return ""
|
||||
}
|
||||
return c.HTTPResponse.Request.Method
|
||||
}
|
||||
|
||||
func (c *Response) closeResponseBody() error {
|
||||
if c.HTTPResponse == nil || c.HTTPResponse.Body == nil {
|
||||
return nil
|
||||
}
|
||||
return c.HTTPResponse.Body.Close()
|
||||
}
|
111
vendor/github.com/cavaliercoder/grab/states.wsd
generated
vendored
Normal file
111
vendor/github.com/cavaliercoder/grab/states.wsd
generated
vendored
Normal file
@ -0,0 +1,111 @@
|
||||
@startuml
|
||||
title Grab transfer state
|
||||
|
||||
legend
|
||||
| # | Meaning |
|
||||
| D | Destination path known |
|
||||
| S | File size known |
|
||||
| O | Server options known (Accept-Ranges) |
|
||||
| R | Resume supported (Accept-Ranges) |
|
||||
| Z | Local file empty or missing |
|
||||
| P | Local file partially complete |
|
||||
endlegend
|
||||
|
||||
[*] --> Empty
|
||||
[*] --> D
|
||||
[*] --> S
|
||||
[*] --> DS
|
||||
|
||||
Empty : Filename: ""
|
||||
Empty : Size: 0
|
||||
Empty --> O : HEAD: Method not allowed
|
||||
Empty --> DSO : HEAD: Range not supported
|
||||
Empty --> DSOR : HEAD: Range supported
|
||||
|
||||
DS : Filename: "foo.bar"
|
||||
DS : Size: > 0
|
||||
DS --> DSZ : checkExisting(): File missing
|
||||
DS --> DSP : checkExisting(): File partial
|
||||
DS --> [*] : checkExisting(): File complete
|
||||
DS --> ERROR
|
||||
|
||||
S : Filename: ""
|
||||
S : Size: > 0
|
||||
S --> SO : HEAD: Method not allowed
|
||||
S --> DSO : HEAD: Range not supported
|
||||
S --> DSOR : HEAD: Range supported
|
||||
|
||||
D : Filename: "foo.bar"
|
||||
D : Size: 0
|
||||
D --> DO : HEAD: Method not allowed
|
||||
D --> DSO : HEAD: Range not supported
|
||||
D --> DSOR : HEAD: Range supported
|
||||
|
||||
|
||||
O : Filename: ""
|
||||
O : Size: 0
|
||||
O : CanResume: false
|
||||
O --> DSO : GET 200
|
||||
O --> ERROR
|
||||
|
||||
SO : Filename: ""
|
||||
SO : Size: > 0
|
||||
SO : CanResume: false
|
||||
SO --> DSO : GET: 200
|
||||
SO --> ERROR
|
||||
|
||||
DO : Filename: "foo.bar"
|
||||
DO : Size: 0
|
||||
DO : CanResume: false
|
||||
DO --> DSO : GET 200
|
||||
DO --> ERROR
|
||||
|
||||
DSZ : Filename: "foo.bar"
|
||||
DSZ : Size: > 0
|
||||
DSZ : File: empty
|
||||
DSZ --> DSORZ : HEAD: Range supported
|
||||
DSZ --> DSOZ : HEAD 405 or Range unsupported
|
||||
|
||||
DSP : Filename: "foo.bar"
|
||||
DSP : Size: > 0
|
||||
DSP : File: partial
|
||||
DSP --> DSORP : HEAD: Range supported
|
||||
DSP --> DSOZ : HEAD: 405 or Range unsupported
|
||||
|
||||
DSO : Filename: "foo.bar"
|
||||
DSO : Size: > 0
|
||||
DSO : CanResume: false
|
||||
DSO --> DSOZ : checkExisting(): File partial|missing
|
||||
DSO --> [*] : checkExisting(): File complete
|
||||
|
||||
DSOR : Filename: "foo.bar"
|
||||
DSOR : Size: > 0
|
||||
DSOR : CanResume: true
|
||||
DSOR --> DSORP : CheckLocal: File partial
|
||||
DSOR --> DSORZ : CheckLocal: File missing
|
||||
|
||||
DSORP : Filename: "foo.bar"
|
||||
DSORP : Size: > 0
|
||||
DSORP : CanResume: true
|
||||
DSORP : File: partial
|
||||
DSORP --> Transferring
|
||||
|
||||
DSORZ : Filename: "foo.bar"
|
||||
DSORZ : Size: > 0
|
||||
DSORZ : CanResume: true
|
||||
DSORZ : File: empty
|
||||
DSORZ --> Transferring
|
||||
|
||||
DSOZ : Filename: "foo.bar"
|
||||
DSOZ : Size: > 0
|
||||
DSOZ : CanResume: false
|
||||
DSOZ : File: empty
|
||||
DSOZ --> Transferring
|
||||
|
||||
Transferring --> [*]
|
||||
Transferring --> ERROR
|
||||
|
||||
ERROR : Something went wrong
|
||||
ERROR --> [*]
|
||||
|
||||
@enduml
|
81
vendor/github.com/cavaliercoder/grab/transfer.go
generated
vendored
Normal file
81
vendor/github.com/cavaliercoder/grab/transfer.go
generated
vendored
Normal file
@ -0,0 +1,81 @@
|
||||
package grab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
type transfer struct {
|
||||
n int64 // must be 64bit aligned on 386
|
||||
ctx context.Context
|
||||
lim RateLimiter
|
||||
w io.Writer
|
||||
r io.Reader
|
||||
b []byte
|
||||
}
|
||||
|
||||
func newTransfer(ctx context.Context, lim RateLimiter, dst io.Writer, src io.Reader, buf []byte) *transfer {
|
||||
return &transfer{
|
||||
ctx: ctx,
|
||||
lim: lim,
|
||||
w: dst,
|
||||
r: src,
|
||||
b: buf,
|
||||
}
|
||||
}
|
||||
|
||||
// copy behaves similarly to io.CopyBuffer except that it checks for cancelation
|
||||
// of the given context.Context and reports progress in a thread-safe manner.
|
||||
func (c *transfer) copy() (written int64, err error) {
|
||||
if c.b == nil {
|
||||
c.b = make([]byte, 32*1024)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-c.ctx.Done():
|
||||
err = c.ctx.Err()
|
||||
return
|
||||
default:
|
||||
// keep working
|
||||
}
|
||||
if c.lim != nil {
|
||||
err = c.lim.WaitN(c.ctx, len(c.b))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
nr, er := c.r.Read(c.b)
|
||||
if nr > 0 {
|
||||
nw, ew := c.w.Write(c.b[0:nr])
|
||||
if nw > 0 {
|
||||
written += int64(nw)
|
||||
atomic.StoreInt64(&c.n, written)
|
||||
}
|
||||
if ew != nil {
|
||||
err = ew
|
||||
break
|
||||
}
|
||||
if nr != nw {
|
||||
err = io.ErrShortWrite
|
||||
break
|
||||
}
|
||||
}
|
||||
if er != nil {
|
||||
if er != io.EOF {
|
||||
err = er
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return written, err
|
||||
}
|
||||
|
||||
// N returns the number of bytes transferred.
|
||||
func (c *transfer) N() (n int64) {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
n = atomic.LoadInt64(&c.n)
|
||||
return
|
||||
}
|
89
vendor/github.com/cavaliercoder/grab/util.go
generated
vendored
Normal file
89
vendor/github.com/cavaliercoder/grab/util.go
generated
vendored
Normal file
@ -0,0 +1,89 @@
|
||||
package grab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"hash"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// setLastModified sets the last modified timestamp of a local file according to
|
||||
// the Last-Modified header returned by a remote server.
|
||||
func setLastModified(resp *http.Response, filename string) error {
|
||||
// https://tools.ietf.org/html/rfc7232#section-2.2
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified
|
||||
header := resp.Header.Get("Last-Modified")
|
||||
if header == "" {
|
||||
return nil
|
||||
}
|
||||
lastmod, err := time.Parse(http.TimeFormat, header)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return os.Chtimes(filename, lastmod, lastmod)
|
||||
}
|
||||
|
||||
// mkdirp creates all missing parent directories for the destination file path.
|
||||
func mkdirp(path string) error {
|
||||
dir := filepath.Dir(path)
|
||||
if fi, err := os.Stat(dir); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("error checking destination directory: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("error creating destination directory: %v", err)
|
||||
}
|
||||
} else if !fi.IsDir() {
|
||||
panic("destination path is not directory")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// guessFilename returns a filename for the given http.Response. If none can be
|
||||
// determined ErrNoFilename is returned.
|
||||
func guessFilename(resp *http.Response) (string, error) {
|
||||
filename := resp.Request.URL.Path
|
||||
if cd := resp.Header.Get("Content-Disposition"); cd != "" {
|
||||
if _, params, err := mime.ParseMediaType(cd); err == nil {
|
||||
filename = params["filename"]
|
||||
}
|
||||
}
|
||||
|
||||
// sanitize
|
||||
if filename == "" || strings.HasSuffix(filename, "/") || strings.Contains(filename, "\x00") {
|
||||
return "", ErrNoFilename
|
||||
}
|
||||
|
||||
filename = filepath.Base(path.Clean("/" + filename))
|
||||
if filename == "" || filename == "." || filename == "/" {
|
||||
return "", ErrNoFilename
|
||||
}
|
||||
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// checksum returns a hash of the given file, using the given hash algorithm.
|
||||
func checksum(ctx context.Context, filename string, h hash.Hash) (b []byte, err error) {
|
||||
var f *os.File
|
||||
f, err = os.Open(filename)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
err = f.Close()
|
||||
}()
|
||||
|
||||
t := newTransfer(ctx, nil, h, f, nil)
|
||||
if _, err = t.copy(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
b = h.Sum(nil)
|
||||
return
|
||||
}
|
2
vendor/modules.txt
vendored
2
vendor/modules.txt
vendored
@ -34,6 +34,8 @@ github.com/asdine/storm/index
|
||||
github.com/asdine/storm/internal
|
||||
# github.com/briandowns/spinner v1.7.0
|
||||
github.com/briandowns/spinner
|
||||
# github.com/cavaliercoder/grab v2.0.0+incompatible
|
||||
github.com/cavaliercoder/grab
|
||||
# github.com/containerd/containerd v1.2.4
|
||||
github.com/containerd/containerd/archive
|
||||
github.com/containerd/containerd/log
|
||||
|
Loading…
Reference in New Issue
Block a user