diff --git a/go.mod b/go.mod index 107e6313..21b54b46 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/containers/common v0.14.0 - github.com/containers/image/v5 v5.4.4 + github.com/containers/image/v5 v5.5.1 github.com/containers/ocicrypt v1.0.2 github.com/containers/storage v1.20.2 github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f diff --git a/go.sum b/go.sum index 0bab549d..a39cc0aa 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/containers/common v0.14.0 h1:hiZFDPf6ajKiDmojN5f5X3gboKPO73NLrYb0RXfr github.com/containers/common v0.14.0/go.mod h1:9olhlE+WhYof1npnMJdyRMX14/yIUint6zyHzcyRVAg= github.com/containers/image/v5 v5.4.4 h1:JSanNn3v/BMd3o0MEvO4R4OKNuoJUSzVGQAI1+0FMXE= github.com/containers/image/v5 v5.4.4/go.mod h1:g7cxNXitiLi6pEr9/L9n/0wfazRuhDKXU15kV86N8h8= +github.com/containers/image/v5 v5.5.1 h1:h1FCOXH6Ux9/p/E4rndsQOC4yAdRU0msRTfLVeQ7FDQ= +github.com/containers/image/v5 v5.5.1/go.mod h1:4PyNYR0nwlGq/ybVJD9hWlhmIsNra4Q8uOQX2s6E2uM= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.2 h1:Q0/IPs8ohfbXNxEfyJ2pFVmvJu5BhqJUAmc6ES9NKbo= @@ -152,6 +154,8 @@ github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqj github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY= +github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.3 h1:Ce2to9wvs/cuJ2b86/CKQoTYr9VHfpanYosZ0UBJqdw= github.com/klauspost/pgzip v1.2.3/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -167,6 +171,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -298,6 +306,8 @@ github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02 github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= github.com/vbauerster/mpb/v5 v5.0.4 h1:w7l/tJfHmtIOKZkU+bhbDZOUxj1kln9jy4DUOp3Tl14= github.com/vbauerster/mpb/v5 v5.0.4/go.mod h1:fvzasBUyuo35UyuA6sSOlVhpLoNQsp2nBdHw7OiSUU8= +github.com/vbauerster/mpb/v5 v5.2.2 h1:zIICVOm+XD+uV6crpSORaL6I0Q1WqOdvxZTp+r3L9cw= +github.com/vbauerster/mpb/v5 v5.2.2/go.mod h1:W5Fvgw4dm3/0NhqzV8j6EacfuTe5SvnzBRwiXxDR9ww= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b h1:6cLsL+2FW6dRAdl5iMtHgRogVCff0QpRi9653YmdcJA= github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -369,6 +379,7 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go index 9fc0e512..7482fb45 100644 --- a/vendor/github.com/containers/image/v5/copy/copy.go +++ b/vendor/github.com/containers/image/v5/copy/copy.go @@ -659,7 +659,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli // With !ic.canModifyManifest, that would just be a string of repeated failures for the same reason, // so let’s bail out early and with a better error message. if !ic.canModifyManifest { - return nil, "", "", errors.Wrap(err, "Writing manifest failed (and converting it is not possible)") + return nil, "", "", errors.Wrap(err, "Writing manifest failed (and converting it is not possible, image is signed or the destination specifies a digest)") } // errs is a list of errors when trying various manifest types. Also serves as an "upload succeeded" flag when set to nil. @@ -757,7 +757,7 @@ func (ic *imageCopier) updateEmbeddedDockerReference() error { } if !ic.canModifyManifest { - return errors.Errorf("Copying a schema1 image with an embedded Docker reference to %s (Docker reference %s) would invalidate existing signatures. Explicitly enable signature removal to proceed anyway", + return errors.Errorf("Copying a schema1 image with an embedded Docker reference to %s (Docker reference %s) would change the manifest, which is not possible (image is signed or the destination specifies a digest)", transports.ImageName(ic.c.dest.Reference()), destRef.String()) } ic.manifestUpdates.EmbeddedDockerReference = destRef @@ -784,7 +784,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { // If we only need to check authorization, no updates required. if updatedSrcInfos != nil && !reflect.DeepEqual(srcInfos, updatedSrcInfos) { if !ic.canModifyManifest { - return errors.Errorf("Internal error: copyLayers() needs to use an updated manifest but that was known to be forbidden") + return errors.Errorf("Copying this image requires changing layer representation, which is not possible (image is signed or the destination specifies a digest)") } srcInfos = updatedSrcInfos srcInfosUpdated = true @@ -1060,6 +1060,14 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to logrus.Debugf("Skipping blob %s (already present):", srcInfo.Digest) bar := ic.c.createProgressBar(pool, srcInfo, "blob", "skipped: already exists") bar.SetTotal(0, true) + + // Throw an event that the layer has been skipped + if ic.c.progress != nil && ic.c.progressInterval > 0 { + ic.c.progress <- types.ProgressProperties{ + Event: types.ProgressEventSkipped, + Artifact: srcInfo, + } + } return blobInfo, cachedDiffID, nil } } diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_src.go b/vendor/github.com/containers/image/v5/docker/docker_image_src.go index 10aff615..55eb3882 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_src.go @@ -190,6 +190,7 @@ func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest strin if err != nil { return nil, "", err } + logrus.Debugf("Content-Type from manifest GET is %q", res.Header.Get("Content-Type")) defer res.Body.Close() if res.StatusCode != http.StatusOK { return nil, "", errors.Wrapf(client.HandleErrorResponse(res), "Error reading manifest %s in %s", tagOrDigest, s.physicalRef.ref.Name()) diff --git a/vendor/github.com/containers/image/v5/manifest/oci.go b/vendor/github.com/containers/image/v5/manifest/oci.go index 40c40dee..29261459 100644 --- a/vendor/github.com/containers/image/v5/manifest/oci.go +++ b/vendor/github.com/containers/image/v5/manifest/oci.go @@ -172,7 +172,7 @@ func (m *OCI1) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*type Architecture: v1.Architecture, Os: v1.OS, Layers: layerInfosToStrings(m.LayerInfos()), - Env: d1.Config.Env, + Env: v1.Config.Env, } return i, nil } diff --git a/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go b/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go index fb0449ca..48a32315 100644 --- a/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go +++ b/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go @@ -279,7 +279,7 @@ func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) { // If it has the same digest as another entry in the index, we already overwrote the file, // so just pick up the other information. for i, manifest := range d.index.Manifests { - if manifest.Digest == desc.Digest { + if manifest.Digest == desc.Digest && manifest.Annotations[imgspecv1.AnnotationRefName] == "" { // Replace it completely. d.index.Manifests[i] = *desc return diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go index dae3eb58..ce85af18 100644 --- a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go +++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "strings" "github.com/containers/image/v5/types" @@ -37,7 +38,12 @@ var ( xdgRuntimeDirPath = filepath.FromSlash("containers/auth.json") dockerHomePath = filepath.FromSlash(".docker/config.json") dockerLegacyHomePath = ".dockercfg" + nonLinuxAuthFilePath = filepath.FromSlash(".config/containers/auth.json") + // Note that the keyring support has been disabled as it was causing + // regressions. Before enabling, please revisit TODO(keyring) comments + // which need to be addressed if the need remerged to support the + // kernel keyring. enableKeyring = false // ErrNotLoggedIn is returned for users not logged into a registry @@ -73,6 +79,70 @@ func SetAuthentication(sys *types.SystemContext, registry, username, password st }) } +// GetAllCredentials returns the registry credentials for all registries stored +// in either the auth.json file or the docker/config.json. +func GetAllCredentials(sys *types.SystemContext) (map[string]types.DockerAuthConfig, error) { + // Note: we need to read the auth files in the inverse order to prevent + // a priority inversion when writing to the map. + authConfigs := make(map[string]types.DockerAuthConfig) + paths := getAuthFilePaths(sys) + for i := len(paths) - 1; i >= 0; i-- { + path := paths[i] + // readJSONFile returns an empty map in case the path doesn't exist. + auths, err := readJSONFile(path.path, path.legacyFormat) + if err != nil { + return nil, errors.Wrapf(err, "error reading JSON file %q", path.path) + } + + for registry, data := range auths.AuthConfigs { + conf, err := decodeDockerAuth(data) + if err != nil { + return nil, err + } + authConfigs[normalizeRegistry(registry)] = conf + } + + // Credential helpers may override credentials from the auth file. + for registry, credHelper := range auths.CredHelpers { + username, password, err := getAuthFromCredHelper(credHelper, registry) + if err != nil { + if credentials.IsErrCredentialsNotFoundMessage(err.Error()) { + continue + } + return nil, err + } + + conf := types.DockerAuthConfig{Username: username, Password: password} + authConfigs[normalizeRegistry(registry)] = conf + } + } + + // TODO(keyring): if we ever reenable the keyring support, we had to + // query all credentials from the keyring here. + + return authConfigs, nil +} + +// getAuthFilePaths returns a slice of authPaths based on the system context +// in the order they should be searched. Note that some paths may not exist. +func getAuthFilePaths(sys *types.SystemContext) []authPath { + paths := []authPath{} + pathToAuth, lf, err := getPathToAuth(sys) + if err == nil { + paths = append(paths, authPath{path: pathToAuth, legacyFormat: lf}) + } else { + // Error means that the path set for XDG_RUNTIME_DIR does not exist + // but we don't want to completely fail in the case that the user is pulling a public image + // Logging the error as a warning instead and moving on to pulling the image + logrus.Warnf("%v: Trying to pull image in the event that it is a public image.", err) + } + paths = append(paths, + authPath{path: filepath.Join(homedir.Get(), dockerHomePath), legacyFormat: false}, + authPath{path: filepath.Join(homedir.Get(), dockerLegacyHomePath), legacyFormat: true}, + ) + return paths +} + // GetCredentials returns the registry credentials stored in either auth.json // file or .docker/config.json, including support for OAuth2 and IdentityToken. // If an entry is not found, an empty struct is returned. @@ -93,21 +163,7 @@ func GetCredentials(sys *types.SystemContext, registry string) (types.DockerAuth } } - paths := []authPath{} - pathToAuth, lf, err := getPathToAuth(sys) - if err == nil { - paths = append(paths, authPath{path: pathToAuth, legacyFormat: lf}) - } else { - // Error means that the path set for XDG_RUNTIME_DIR does not exist - // but we don't want to completely fail in the case that the user is pulling a public image - // Logging the error as a warning instead and moving on to pulling the image - logrus.Warnf("%v: Trying to pull image in the event that it is a public image.", err) - } - paths = append(paths, - authPath{path: filepath.Join(homedir.Get(), dockerHomePath), legacyFormat: false}, - authPath{path: filepath.Join(homedir.Get(), dockerLegacyHomePath), legacyFormat: true}) - - for _, path := range paths { + for _, path := range getAuthFilePaths(sys) { authConfig, err := findAuthentication(registry, path.path, path.legacyFormat) if err != nil { logrus.Debugf("Credentials not found") @@ -189,10 +245,8 @@ func RemoveAllAuthentication(sys *types.SystemContext) error { }) } -// getPath gets the path of the auth.json file -// The path can be overriden by the user if the overwrite-path flag is set -// If the flag is not set and XDG_RUNTIME_DIR is set, the auth.json file is saved in XDG_RUNTIME_DIR/containers -// Otherwise, the auth.json file is stored in /run/containers/UID +// getPathToAuth gets the path of the auth.json file used for reading and writting credentials +// returns the path, and a bool specifies whether the file is in legacy format func getPathToAuth(sys *types.SystemContext) (string, bool, error) { if sys != nil { if sys.AuthFilePath != "" { @@ -205,6 +259,9 @@ func getPathToAuth(sys *types.SystemContext) (string, bool, error) { return filepath.Join(sys.RootForImplicitAbsolutePaths, fmt.Sprintf(defaultPerUIDPathFormat, os.Getuid())), false, nil } } + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + return filepath.Join(homedir.Get(), nonLinuxAuthFilePath), false, nil + } runtimeDir := os.Getenv("XDG_RUNTIME_DIR") if runtimeDir != "" { @@ -248,6 +305,13 @@ func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) { return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path) } + if auths.AuthConfigs == nil { + auths.AuthConfigs = map[string]dockerAuthConfig{} + } + if auths.CredHelpers == nil { + auths.CredHelpers = make(map[string]string) + } + return auths, nil } @@ -257,17 +321,15 @@ func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) ( if err != nil { return err } - - dir := filepath.Dir(path) - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err = os.MkdirAll(dir, 0700); err != nil { - return errors.Wrapf(err, "error creating directory %q", dir) - } - } - if legacyFormat { return fmt.Errorf("writes to %s using legacy format are not supported", path) } + + dir := filepath.Dir(path) + if err = os.MkdirAll(dir, 0700); err != nil { + return err + } + auths, err := readJSONFile(path, false) if err != nil { return errors.Wrapf(err, "error reading JSON file %q", path) diff --git a/vendor/github.com/containers/image/v5/signature/policy_config.go b/vendor/github.com/containers/image/v5/signature/policy_config.go index 3eee70bc..72fcf618 100644 --- a/vendor/github.com/containers/image/v5/signature/policy_config.go +++ b/vendor/github.com/containers/image/v5/signature/policy_config.go @@ -17,11 +17,13 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path/filepath" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" + "github.com/containers/storage/pkg/homedir" "github.com/pkg/errors" ) @@ -34,6 +36,9 @@ var systemDefaultPolicyPath = builtinDefaultPolicyPath // DO NOT change this, instead see systemDefaultPolicyPath above. const builtinDefaultPolicyPath = "/etc/containers/policy.json" +// userPolicyFile is the path to the per user policy path. +var userPolicyFile = filepath.FromSlash(".config/containers/policy.json") + // InvalidPolicyFormatError is returned when parsing an invalid policy configuration. type InvalidPolicyFormatError string @@ -53,13 +58,15 @@ func DefaultPolicy(sys *types.SystemContext) (*Policy, error) { // defaultPolicyPath returns a path to the default policy of the system. func defaultPolicyPath(sys *types.SystemContext) string { - if sys != nil { - if sys.SignaturePolicyPath != "" { - return sys.SignaturePolicyPath - } - if sys.RootForImplicitAbsolutePaths != "" { - return filepath.Join(sys.RootForImplicitAbsolutePaths, systemDefaultPolicyPath) - } + if sys != nil && sys.SignaturePolicyPath != "" { + return sys.SignaturePolicyPath + } + userPolicyFilePath := filepath.Join(homedir.Get(), userPolicyFile) + if _, err := os.Stat(userPolicyFilePath); err == nil { + return userPolicyFilePath + } + if sys != nil && sys.RootForImplicitAbsolutePaths != "" { + return filepath.Join(sys.RootForImplicitAbsolutePaths, systemDefaultPolicyPath) } return systemDefaultPolicyPath } diff --git a/vendor/github.com/containers/image/v5/types/types.go b/vendor/github.com/containers/image/v5/types/types.go index d469e03b..4f624cf3 100644 --- a/vendor/github.com/containers/image/v5/types/types.go +++ b/vendor/github.com/containers/image/v5/types/types.go @@ -604,6 +604,10 @@ const ( // ProgressEventDone is fired when the data transfer has been finished for // the specific artifact ProgressEventDone + + // ProgressEventSkipped is fired when the artifact has been skipped because + // its already available at the destination + ProgressEventSkipped ) // ProgressProperties is used to pass information from the copy code to a monitor which diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index 67f57e03..114bce38 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -6,9 +6,9 @@ const ( // VersionMajor is for an API incompatible changes VersionMajor = 5 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 4 + VersionMinor = 5 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 4 + VersionPatch = 1 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/github.com/klauspost/compress/fse/bitreader.go b/vendor/github.com/klauspost/compress/fse/bitreader.go index b9db204f..f65eb390 100644 --- a/vendor/github.com/klauspost/compress/fse/bitreader.go +++ b/vendor/github.com/klauspost/compress/fse/bitreader.go @@ -6,6 +6,7 @@ package fse import ( + "encoding/binary" "errors" "io" ) @@ -34,8 +35,12 @@ func (b *bitReader) init(in []byte) error { } b.bitsRead = 64 b.value = 0 - b.fill() - b.fill() + if len(in) >= 8 { + b.fillFastStart() + } else { + b.fill() + b.fill() + } b.bitsRead += 8 - uint8(highBits(uint32(v))) return nil } @@ -63,8 +68,9 @@ func (b *bitReader) fillFast() { if b.bitsRead < 32 { return } - // Do single re-slice to avoid bounds checks. - v := b.in[b.off-4 : b.off] + // 2 bounds checks. + v := b.in[b.off-4:] + v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value = (b.value << 32) | uint64(low) b.bitsRead -= 32 @@ -77,7 +83,8 @@ func (b *bitReader) fill() { return } if b.off > 4 { - v := b.in[b.off-4 : b.off] + v := b.in[b.off-4:] + v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value = (b.value << 32) | uint64(low) b.bitsRead -= 32 @@ -91,9 +98,17 @@ func (b *bitReader) fill() { } } +// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. +func (b *bitReader) fillFastStart() { + // Do single re-slice to avoid bounds checks. + b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) + b.bitsRead = 0 + b.off -= 8 +} + // finished returns true if all bits have been read from the bit stream. func (b *bitReader) finished() bool { - return b.off == 0 && b.bitsRead >= 64 + return b.bitsRead >= 64 && b.off == 0 } // close the bitstream and returns an error if out-of-buffer reads occurred. diff --git a/vendor/github.com/klauspost/compress/fse/bytereader.go b/vendor/github.com/klauspost/compress/fse/bytereader.go index f228a46c..abade2d6 100644 --- a/vendor/github.com/klauspost/compress/fse/bytereader.go +++ b/vendor/github.com/klauspost/compress/fse/bytereader.go @@ -25,19 +25,10 @@ func (b *byteReader) advance(n uint) { b.off += int(n) } -// Int32 returns a little endian int32 starting at current offset. -func (b byteReader) Int32() int32 { - b2 := b.b[b.off : b.off+4 : b.off+4] - v3 := int32(b2[3]) - v2 := int32(b2[2]) - v1 := int32(b2[1]) - v0 := int32(b2[0]) - return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) -} - // Uint32 returns a little endian uint32 starting at current offset. func (b byteReader) Uint32() uint32 { - b2 := b.b[b.off : b.off+4 : b.off+4] + b2 := b.b[b.off:] + b2 = b2[:4] v3 := uint32(b2[3]) v2 := uint32(b2[2]) v1 := uint32(b2[1]) diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md index 0a8448ce..e12da4db 100644 --- a/vendor/github.com/klauspost/compress/huff0/README.md +++ b/vendor/github.com/klauspost/compress/huff0/README.md @@ -12,8 +12,6 @@ but it can be used as a secondary step to compressors (like Snappy) that does no * [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) -THIS PACKAGE IS NOT CONSIDERED STABLE AND API OR ENCODING MAY CHANGE IN THE FUTURE. - ## News * Mar 2018: First implementation released. Consider this beta software for now. @@ -75,6 +73,8 @@ which can be given to the decompressor. Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. +For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. + You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back your input was likely corrupted. @@ -84,4 +84,4 @@ There are no integrity checks, so relying on errors from the decompressor does n # Contributing Contributions are always welcome. Be aware that adding public functions will require good justification and breaking -changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file +changes will likely not be accepted. If in doubt open an issue before writing the PR. diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go index 7d0903c7..a4979e88 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -6,6 +6,7 @@ package huff0 import ( + "encoding/binary" "errors" "io" ) @@ -34,29 +35,16 @@ func (b *bitReader) init(in []byte) error { } b.bitsRead = 64 b.value = 0 - b.fill() - b.fill() + if len(in) >= 8 { + b.fillFastStart() + } else { + b.fill() + b.fill() + } b.bitsRead += 8 - uint8(highBit32(uint32(v))) return nil } -// getBits will return n bits. n can be 0. -func (b *bitReader) getBits(n uint8) uint16 { - if n == 0 || b.bitsRead >= 64 { - return 0 - } - return b.getBitsFast(n) -} - -// getBitsFast requires that at least one bit is requested every time. -// There are no checks if the buffer is filled. -func (b *bitReader) getBitsFast(n uint8) uint16 { - const regMask = 64 - 1 - v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) - b.bitsRead += n - return v -} - // peekBitsFast requires that at least one bit is requested every time. // There are no checks if the buffer is filled. func (b *bitReader) peekBitsFast(n uint8) uint16 { @@ -71,21 +59,36 @@ func (b *bitReader) fillFast() { if b.bitsRead < 32 { return } - // Do single re-slice to avoid bounds checks. + + // 2 bounds checks. v := b.in[b.off-4 : b.off] + v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value = (b.value << 32) | uint64(low) b.bitsRead -= 32 b.off -= 4 } +func (b *bitReader) advance(n uint8) { + b.bitsRead += n +} + +// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. +func (b *bitReader) fillFastStart() { + // Do single re-slice to avoid bounds checks. + b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) + b.bitsRead = 0 + b.off -= 8 +} + // fill() will make sure at least 32 bits are available. func (b *bitReader) fill() { if b.bitsRead < 32 { return } if b.off > 4 { - v := b.in[b.off-4 : b.off] + v := b.in[b.off-4:] + v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value = (b.value << 32) | uint64(low) b.bitsRead -= 32 @@ -113,3 +116,214 @@ func (b *bitReader) close() error { } return nil } + +// bitReader reads a bitstream in reverse. +// The last set bit indicates the start of the stream and is used +// for aligning the input. +type bitReaderBytes struct { + in []byte + off uint // next byte to read is at in[off - 1] + value uint64 + bitsRead uint8 +} + +// init initializes and resets the bit reader. +func (b *bitReaderBytes) init(in []byte) error { + if len(in) < 1 { + return errors.New("corrupt stream: too short") + } + b.in = in + b.off = uint(len(in)) + // The highest bit of the last byte indicates where to start + v := in[len(in)-1] + if v == 0 { + return errors.New("corrupt stream, did not find end of stream") + } + b.bitsRead = 64 + b.value = 0 + if len(in) >= 8 { + b.fillFastStart() + } else { + b.fill() + b.fill() + } + b.advance(8 - uint8(highBit32(uint32(v)))) + return nil +} + +// peekBitsFast requires that at least one bit is requested every time. +// There are no checks if the buffer is filled. +func (b *bitReaderBytes) peekByteFast() uint8 { + got := uint8(b.value >> 56) + return got +} + +func (b *bitReaderBytes) advance(n uint8) { + b.bitsRead += n + b.value <<= n & 63 +} + +// fillFast() will make sure at least 32 bits are available. +// There must be at least 4 bytes available. +func (b *bitReaderBytes) fillFast() { + if b.bitsRead < 32 { + return + } + + // 2 bounds checks. + v := b.in[b.off-4 : b.off] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value |= uint64(low) << (b.bitsRead - 32) + b.bitsRead -= 32 + b.off -= 4 +} + +// fillFastStart() assumes the bitReaderBytes is empty and there is at least 8 bytes to read. +func (b *bitReaderBytes) fillFastStart() { + // Do single re-slice to avoid bounds checks. + b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) + b.bitsRead = 0 + b.off -= 8 +} + +// fill() will make sure at least 32 bits are available. +func (b *bitReaderBytes) fill() { + if b.bitsRead < 32 { + return + } + if b.off > 4 { + v := b.in[b.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value |= uint64(low) << (b.bitsRead - 32) + b.bitsRead -= 32 + b.off -= 4 + return + } + for b.off > 0 { + b.value |= uint64(b.in[b.off-1]) << (b.bitsRead - 8) + b.bitsRead -= 8 + b.off-- + } +} + +// finished returns true if all bits have been read from the bit stream. +func (b *bitReaderBytes) finished() bool { + return b.off == 0 && b.bitsRead >= 64 +} + +// close the bitstream and returns an error if out-of-buffer reads occurred. +func (b *bitReaderBytes) close() error { + // Release reference. + b.in = nil + if b.bitsRead > 64 { + return io.ErrUnexpectedEOF + } + return nil +} + +// bitReaderShifted reads a bitstream in reverse. +// The last set bit indicates the start of the stream and is used +// for aligning the input. +type bitReaderShifted struct { + in []byte + off uint // next byte to read is at in[off - 1] + value uint64 + bitsRead uint8 +} + +// init initializes and resets the bit reader. +func (b *bitReaderShifted) init(in []byte) error { + if len(in) < 1 { + return errors.New("corrupt stream: too short") + } + b.in = in + b.off = uint(len(in)) + // The highest bit of the last byte indicates where to start + v := in[len(in)-1] + if v == 0 { + return errors.New("corrupt stream, did not find end of stream") + } + b.bitsRead = 64 + b.value = 0 + if len(in) >= 8 { + b.fillFastStart() + } else { + b.fill() + b.fill() + } + b.advance(8 - uint8(highBit32(uint32(v)))) + return nil +} + +// peekBitsFast requires that at least one bit is requested every time. +// There are no checks if the buffer is filled. +func (b *bitReaderShifted) peekBitsFast(n uint8) uint16 { + return uint16(b.value >> ((64 - n) & 63)) +} + +func (b *bitReaderShifted) advance(n uint8) { + b.bitsRead += n + b.value <<= n & 63 +} + +// fillFast() will make sure at least 32 bits are available. +// There must be at least 4 bytes available. +func (b *bitReaderShifted) fillFast() { + if b.bitsRead < 32 { + return + } + + // 2 bounds checks. + v := b.in[b.off-4 : b.off] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value |= uint64(low) << ((b.bitsRead - 32) & 63) + b.bitsRead -= 32 + b.off -= 4 +} + +// fillFastStart() assumes the bitReaderShifted is empty and there is at least 8 bytes to read. +func (b *bitReaderShifted) fillFastStart() { + // Do single re-slice to avoid bounds checks. + b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) + b.bitsRead = 0 + b.off -= 8 +} + +// fill() will make sure at least 32 bits are available. +func (b *bitReaderShifted) fill() { + if b.bitsRead < 32 { + return + } + if b.off > 4 { + v := b.in[b.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value |= uint64(low) << ((b.bitsRead - 32) & 63) + b.bitsRead -= 32 + b.off -= 4 + return + } + for b.off > 0 { + b.value |= uint64(b.in[b.off-1]) << ((b.bitsRead - 8) & 63) + b.bitsRead -= 8 + b.off-- + } +} + +// finished returns true if all bits have been read from the bit stream. +func (b *bitReaderShifted) finished() bool { + return b.off == 0 && b.bitsRead >= 64 +} + +// close the bitstream and returns an error if out-of-buffer reads occurred. +func (b *bitReaderShifted) close() error { + // Release reference. + b.in = nil + if b.bitsRead > 64 { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index fb42a398..a03b2634 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -25,6 +25,9 @@ type dEntryDouble struct { len uint8 } +// Uses special code for all tables that are < 8 bits. +const use8BitTables = true + // ReadTable will read a table from the input. // The size of the input may be larger than the table definition. // Any content remaining after the table definition will be returned. @@ -83,6 +86,7 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { } v2 := v & 15 rankStats[v2]++ + // (1 << (v2-1)) is slower since the compiler cannot prove that v2 isn't 0. weightTotal += (1 << v2) >> 1 } if weightTotal == 0 { @@ -142,12 +146,14 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { d := dEntrySingle{ entry: uint16(s.actualTableLog+1-w) | (uint16(n) << 8), } - single := s.dt.single[rankStats[w] : rankStats[w]+length] + rank := &rankStats[w] + single := s.dt.single[*rank : *rank+length] for i := range single { single[i] = d } - rankStats[w] += length + *rank += length } + return s, in, nil } @@ -208,7 +214,10 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { if len(d.dt.single) == 0 { return nil, errors.New("no table loaded") } - var br bitReader + if use8BitTables && d.actualTableLog <= 8 { + return d.decompress1X8Bit(dst, src) + } + var br bitReaderShifted err := br.init(src) if err != nil { return dst, err @@ -216,17 +225,6 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { maxDecodedSize := cap(dst) dst = dst[:0] - decode := func() byte { - val := br.peekBitsFast(d.actualTableLog) /* note : actualTableLog >= 1 */ - v := d.dt.single[val] - br.bitsRead += uint8(v.entry) - return uint8(v.entry >> 8) - } - hasDec := func(v dEntrySingle) byte { - br.bitsRead += uint8(v.entry) - return uint8(v.entry >> 8) - } - // Avoid bounds check by always having full sized table. const tlSize = 1 << tableLogMax const tlMask = tlSize - 1 @@ -238,11 +236,25 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { for br.off >= 8 { br.fillFast() - buf[off+0] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) - buf[off+1] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) + v := dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + // Refill br.fillFast() - buf[off+2] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) - buf[off+3] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + off += 4 if off == 0 { if len(dst)+256 > maxDecodedSize { @@ -259,13 +271,196 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { } dst = append(dst, buf[:off]...) - for !br.finished() { + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { br.fill() + if false && br.bitsRead >= 32 { + if br.off >= 4 { + v := br.in[br.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + br.value = (br.value << 32) | uint64(low) + br.bitsRead -= 32 + br.off -= 4 + } else { + for br.off > 0 { + br.value = (br.value << 8) | uint64(br.in[br.off-1]) + br.bitsRead -= 8 + br.off-- + } + } + } if len(dst) >= maxDecodedSize { br.close() return nil, ErrMaxDecodedSizeExceeded } - dst = append(dst, decode()) + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + return dst, br.close() +} + +// decompress1X8Bit will decompress a 1X encoded stream with tablelog <= 8. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) { + if d.actualTableLog == 8 { + return d.decompress1X8BitExactly(dst, src) + } + var br bitReaderBytes + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:0] + + // Avoid bounds check by always having full sized table. + dt := d.dt.single[:256] + + // Use temp table to avoid bound checks/append penalty. + var buf [256]byte + var off uint8 + + shift := (8 - d.actualTableLog) & 7 + + //fmt.Printf("mask: %b, tl:%d\n", mask, d.actualTableLog) + for br.off >= 4 { + br.fillFast() + v := dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + + if len(dst)+int(off) > maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:off]...) + + // br < 4, so uint8 is fine + bitsLeft := int8(uint8(br.off)*8 + (64 - br.bitsRead)) + for bitsLeft > 0 { + if br.bitsRead >= 64-8 { + for br.off > 0 { + br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) + br.bitsRead -= 8 + br.off-- + } + } + if len(dst) >= maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := dt[br.peekByteFast()>>shift] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= int8(nBits) + dst = append(dst, uint8(v.entry>>8)) + } + return dst, br.close() +} + +// decompress1X8Bit will decompress a 1X encoded stream with tablelog <= 8. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) { + var br bitReaderBytes + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:0] + + // Avoid bounds check by always having full sized table. + dt := d.dt.single[:256] + + // Use temp table to avoid bound checks/append penalty. + var buf [256]byte + var off uint8 + + const shift = 0 + + //fmt.Printf("mask: %b, tl:%d\n", mask, d.actualTableLog) + for br.off >= 4 { + br.fillFast() + v := dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[br.peekByteFast()>>shift] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + + if len(dst)+int(off) > maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:off]...) + + // br < 4, so uint8 is fine + bitsLeft := int8(uint8(br.off)*8 + (64 - br.bitsRead)) + for bitsLeft > 0 { + if br.bitsRead >= 64-8 { + for br.off > 0 { + br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) + br.bitsRead -= 8 + br.off-- + } + } + if len(dst) >= maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := dt[br.peekByteFast()>>shift] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= int8(nBits) + dst = append(dst, uint8(v.entry>>8)) } return dst, br.close() } @@ -274,15 +469,18 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { // The length of the supplied input must match the end of a block exactly. // The *capacity* of the dst slice must match the destination size of // the uncompressed data exactly. -func (s *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(s.dt.single) == 0 { +func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { return nil, errors.New("no table loaded") } if len(src) < 6+(4*1) { return nil, errors.New("input too small") } + if use8BitTables && d.actualTableLog <= 8 { + return d.decompress4X8bit(dst, src) + } - var br [4]bitReader + var br [4]bitReaderShifted start := 6 for i := 0; i < 3; i++ { length := int(src[i*2]) | (int(src[i*2+1]) << 8) @@ -308,14 +506,7 @@ func (s *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { const tlSize = 1 << tableLogMax const tlMask = tlSize - 1 - single := s.dt.single[:tlSize] - - decode := func(br *bitReader) byte { - val := br.peekBitsFast(s.actualTableLog) /* note : actualTableLog >= 1 */ - v := single[val&tlMask] - br.bitsRead += uint8(v.entry) - return uint8(v.entry >> 8) - } + single := d.dt.single[:tlSize] // Use temp table to avoid bound checks/append penalty. var buf [256]byte @@ -324,66 +515,63 @@ func (s *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { // Decode 2 values from each decoder/loop. const bufoff = 256 / 4 -bigloop: for { - for i := range br { - br := &br[i] - if br.off < 4 { - break bigloop - } - br.fillFast() + if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { + break } { const stream = 0 - val := br[stream].peekBitsFast(s.actualTableLog) + const stream2 = 1 + br[stream].fillFast() + br[stream2].fillFast() + + val := br[stream].peekBitsFast(d.actualTableLog) v := single[val&tlMask] - br[stream].bitsRead += uint8(v.entry) - - val2 := br[stream].peekBitsFast(s.actualTableLog) - v2 := single[val2&tlMask] - buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + br[stream].advance(uint8(v.entry)) buf[off+bufoff*stream] = uint8(v.entry >> 8) - br[stream].bitsRead += uint8(v2.entry) - } - { - const stream = 1 - val := br[stream].peekBitsFast(s.actualTableLog) - v := single[val&tlMask] - br[stream].bitsRead += uint8(v.entry) - - val2 := br[stream].peekBitsFast(s.actualTableLog) + val2 := br[stream2].peekBitsFast(d.actualTableLog) v2 := single[val2&tlMask] - buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) - buf[off+bufoff*stream] = uint8(v.entry >> 8) - br[stream].bitsRead += uint8(v2.entry) + br[stream2].advance(uint8(v2.entry)) + buf[off+bufoff*stream2] = uint8(v2.entry >> 8) + + val = br[stream].peekBitsFast(d.actualTableLog) + v = single[val&tlMask] + br[stream].advance(uint8(v.entry)) + buf[off+bufoff*stream+1] = uint8(v.entry >> 8) + + val2 = br[stream2].peekBitsFast(d.actualTableLog) + v2 = single[val2&tlMask] + br[stream2].advance(uint8(v2.entry)) + buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8) } { const stream = 2 - val := br[stream].peekBitsFast(s.actualTableLog) + const stream2 = 3 + br[stream].fillFast() + br[stream2].fillFast() + + val := br[stream].peekBitsFast(d.actualTableLog) v := single[val&tlMask] - br[stream].bitsRead += uint8(v.entry) - - val2 := br[stream].peekBitsFast(s.actualTableLog) - v2 := single[val2&tlMask] - buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + br[stream].advance(uint8(v.entry)) buf[off+bufoff*stream] = uint8(v.entry >> 8) - br[stream].bitsRead += uint8(v2.entry) - } - { - const stream = 3 - val := br[stream].peekBitsFast(s.actualTableLog) - v := single[val&tlMask] - br[stream].bitsRead += uint8(v.entry) - - val2 := br[stream].peekBitsFast(s.actualTableLog) + val2 := br[stream2].peekBitsFast(d.actualTableLog) v2 := single[val2&tlMask] - buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) - buf[off+bufoff*stream] = uint8(v.entry >> 8) - br[stream].bitsRead += uint8(v2.entry) + br[stream2].advance(uint8(v2.entry)) + buf[off+bufoff*stream2] = uint8(v2.entry >> 8) + + val = br[stream].peekBitsFast(d.actualTableLog) + v = single[val&tlMask] + br[stream].advance(uint8(v.entry)) + buf[off+bufoff*stream+1] = uint8(v.entry >> 8) + + val2 = br[stream2].peekBitsFast(d.actualTableLog) + v2 = single[val2&tlMask] + br[stream2].advance(uint8(v2.entry)) + buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8) } off += 2 @@ -422,12 +610,456 @@ bigloop: for i := range br { offset := dstEvery * i br := &br[i] - for !br.finished() { + bitsLeft := br.off*8 + uint(64-br.bitsRead) + for bitsLeft > 0 { br.fill() + if false && br.bitsRead >= 32 { + if br.off >= 4 { + v := br.in[br.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + br.value = (br.value << 32) | uint64(low) + br.bitsRead -= 32 + br.off -= 4 + } else { + for br.off > 0 { + br.value = (br.value << 8) | uint64(br.in[br.off-1]) + br.bitsRead -= 8 + br.off-- + } + } + } + // end inline... if offset >= len(out) { return nil, errors.New("corruption detected: stream overrun 4") } - out[offset] = decode(br) + + // Read value and increment offset. + val := br.peekBitsFast(d.actualTableLog) + v := single[val&tlMask].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= uint(nBits) + out[offset] = uint8(v >> 8) + offset++ + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return dst, nil +} + +// Decompress4X will decompress a 4X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { + if d.actualTableLog == 8 { + return d.decompress4X8bitExactly(dst, src) + } + + var br [4]bitReaderBytes + start := 6 + for i := 0; i < 3; i++ { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { + return nil, errors.New("truncated input (or invalid offset)") + } + err := br[i].init(src[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err := br[3].init(src[start:]) + if err != nil { + return nil, err + } + + // destination, offset to match first output + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst + dstEvery := (dstSize + 3) / 4 + + shift := (8 - d.actualTableLog) & 7 + + const tlSize = 1 << 8 + const tlMask = tlSize - 1 + single := d.dt.single[:tlSize] + + // Use temp table to avoid bound checks/append penalty. + var buf [256]byte + var off uint8 + var decoded int + + // Decode 4 values from each decoder/loop. + const bufoff = 256 / 4 + for { + if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { + break + } + + { + // Interleave 2 decodes. + const stream = 0 + const stream2 = 1 + br[stream].fillFast() + br[stream2].fillFast() + + v := single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 := single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+1] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+1] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+2] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+3] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+3] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + } + + { + const stream = 2 + const stream2 = 3 + br[stream].fillFast() + br[stream2].fillFast() + + v := single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 := single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+1] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+1] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+2] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+3] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+3] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + } + + off += 4 + + if off == bufoff { + if bufoff > dstEvery { + return nil, errors.New("corruption detected: stream overrun 1") + } + copy(out, buf[:bufoff]) + copy(out[dstEvery:], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) + off = 0 + out = out[bufoff:] + decoded += 256 + // There must at least be 3 buffers left. + if len(out) < dstEvery*3 { + return nil, errors.New("corruption detected: stream overrun 2") + } + } + } + if off > 0 { + ioff := int(off) + if len(out) < dstEvery*3+ioff { + return nil, errors.New("corruption detected: stream overrun 3") + } + copy(out, buf[:off]) + copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) + decoded += int(off) * 4 + out = out[off:] + } + + // Decode remaining. + for i := range br { + offset := dstEvery * i + br := &br[i] + bitsLeft := int(br.off*8) + int(64-br.bitsRead) + for bitsLeft > 0 { + if br.finished() { + return nil, io.ErrUnexpectedEOF + } + if br.bitsRead >= 56 { + if br.off >= 4 { + v := br.in[br.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + br.value |= uint64(low) << (br.bitsRead - 32) + br.bitsRead -= 32 + br.off -= 4 + } else { + for br.off > 0 { + br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) + br.bitsRead -= 8 + br.off-- + } + } + } + // end inline... + if offset >= len(out) { + return nil, errors.New("corruption detected: stream overrun 4") + } + + // Read value and increment offset. + v := single[br.peekByteFast()>>shift].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= int(nBits) + out[offset] = uint8(v >> 8) + offset++ + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return dst, nil +} + +// Decompress4X will decompress a 4X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { + var br [4]bitReaderBytes + start := 6 + for i := 0; i < 3; i++ { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { + return nil, errors.New("truncated input (or invalid offset)") + } + err := br[i].init(src[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err := br[3].init(src[start:]) + if err != nil { + return nil, err + } + + // destination, offset to match first output + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst + dstEvery := (dstSize + 3) / 4 + + const shift = 0 + const tlSize = 1 << 8 + const tlMask = tlSize - 1 + single := d.dt.single[:tlSize] + + // Use temp table to avoid bound checks/append penalty. + var buf [256]byte + var off uint8 + var decoded int + + // Decode 4 values from each decoder/loop. + const bufoff = 256 / 4 + for { + if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { + break + } + + { + // Interleave 2 decodes. + const stream = 0 + const stream2 = 1 + br[stream].fillFast() + br[stream2].fillFast() + + v := single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 := single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+1] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+1] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+2] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+3] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+3] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + } + + { + const stream = 2 + const stream2 = 3 + br[stream].fillFast() + br[stream2].fillFast() + + v := single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 := single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+1] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+1] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+2] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+2] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + + v = single[br[stream].peekByteFast()>>shift].entry + buf[off+bufoff*stream+3] = uint8(v >> 8) + br[stream].advance(uint8(v)) + + v2 = single[br[stream2].peekByteFast()>>shift].entry + buf[off+bufoff*stream2+3] = uint8(v2 >> 8) + br[stream2].advance(uint8(v2)) + } + + off += 4 + + if off == bufoff { + if bufoff > dstEvery { + return nil, errors.New("corruption detected: stream overrun 1") + } + copy(out, buf[:bufoff]) + copy(out[dstEvery:], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) + off = 0 + out = out[bufoff:] + decoded += 256 + // There must at least be 3 buffers left. + if len(out) < dstEvery*3 { + return nil, errors.New("corruption detected: stream overrun 2") + } + } + } + if off > 0 { + ioff := int(off) + if len(out) < dstEvery*3+ioff { + return nil, errors.New("corruption detected: stream overrun 3") + } + copy(out, buf[:off]) + copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) + decoded += int(off) * 4 + out = out[off:] + } + + // Decode remaining. + for i := range br { + offset := dstEvery * i + br := &br[i] + bitsLeft := int(br.off*8) + int(64-br.bitsRead) + for bitsLeft > 0 { + if br.finished() { + return nil, io.ErrUnexpectedEOF + } + if br.bitsRead >= 56 { + if br.off >= 4 { + v := br.in[br.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + br.value |= uint64(low) << (br.bitsRead - 32) + br.bitsRead -= 32 + br.off -= 4 + } else { + for br.off > 0 { + br.value |= uint64(br.in[br.off-1]) << (br.bitsRead - 8) + br.bitsRead -= 8 + br.off-- + } + } + } + // end inline... + if offset >= len(out) { + return nil, errors.New("corruption detected: stream overrun 4") + } + + // Read value and increment offset. + v := single[br.peekByteFast()>>shift].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= int(nBits) + out[offset] = uint8(v >> 8) offset++ } decoded += offset - dstEvery*i diff --git a/vendor/github.com/klauspost/compress/zstd/bitreader.go b/vendor/github.com/klauspost/compress/zstd/bitreader.go index 15d79d43..85445853 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitreader.go +++ b/vendor/github.com/klauspost/compress/zstd/bitreader.go @@ -5,6 +5,7 @@ package zstd import ( + "encoding/binary" "errors" "io" "math/bits" @@ -34,8 +35,12 @@ func (b *bitReader) init(in []byte) error { } b.bitsRead = 64 b.value = 0 - b.fill() - b.fill() + if len(in) >= 8 { + b.fillFastStart() + } else { + b.fill() + b.fill() + } b.bitsRead += 8 - uint8(highBits(uint32(v))) return nil } @@ -63,21 +68,31 @@ func (b *bitReader) fillFast() { if b.bitsRead < 32 { return } - // Do single re-slice to avoid bounds checks. - v := b.in[b.off-4 : b.off] + // 2 bounds checks. + v := b.in[b.off-4:] + v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value = (b.value << 32) | uint64(low) b.bitsRead -= 32 b.off -= 4 } +// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. +func (b *bitReader) fillFastStart() { + // Do single re-slice to avoid bounds checks. + b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) + b.bitsRead = 0 + b.off -= 8 +} + // fill() will make sure at least 32 bits are available. func (b *bitReader) fill() { if b.bitsRead < 32 { return } if b.off >= 4 { - v := b.in[b.off-4 : b.off] + v := b.in[b.off-4:] + v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value = (b.value << 32) | uint64(low) b.bitsRead -= 32 diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 4a14242c..c8ec6e33 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -83,6 +83,10 @@ type blockDec struct { err error decWG sync.WaitGroup + // Frame to use for singlethreaded decoding. + // Should not be used by the decoder itself since parent may be another frame. + localFrame *frameDec + // Block is RLE, this is the size. RLESize uint32 tmp [4]byte diff --git a/vendor/github.com/klauspost/compress/zstd/bytereader.go b/vendor/github.com/klauspost/compress/zstd/bytereader.go index f708df1c..2c4fca17 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytereader.go +++ b/vendor/github.com/klauspost/compress/zstd/bytereader.go @@ -4,8 +4,6 @@ package zstd -import "encoding/binary" - // byteReader provides a byte reader that reads // little endian values from a byte stream. // The input stream is manually advanced. @@ -33,7 +31,8 @@ func (b *byteReader) overread() bool { // Int32 returns a little endian int32 starting at current offset. func (b byteReader) Int32() int32 { - b2 := b.b[b.off : b.off+4 : b.off+4] + b2 := b.b[b.off:] + b2 = b2[:4] v3 := int32(b2[3]) v2 := int32(b2[2]) v1 := int32(b2[1]) @@ -57,7 +56,25 @@ func (b byteReader) Uint32() uint32 { } return v } - return binary.LittleEndian.Uint32(b.b[b.off : b.off+4]) + b2 := b.b[b.off:] + b2 = b2[:4] + v3 := uint32(b2[3]) + v2 := uint32(b2[2]) + v1 := uint32(b2[1]) + v0 := uint32(b2[0]) + return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) +} + +// Uint32NC returns a little endian uint32 starting at current offset. +// The caller must be sure if there are at least 4 bytes left. +func (b byteReader) Uint32NC() uint32 { + b2 := b.b[b.off:] + b2 = b2[:4] + v3 := uint32(b2[3]) + v2 := uint32(b2[2]) + v1 := uint32(b2[1]) + v0 := uint32(b2[0]) + return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) } // unread returns the unread portion of the input. diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 8e34479f..75bf05bc 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -23,9 +23,6 @@ type Decoder struct { // Unreferenced decoders, ready for use. decoders chan *blockDec - // Unreferenced decoders, ready for use. - frames chan *frameDec - // Streams ready to be decoded. stream chan decodeStream @@ -90,10 +87,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { // Create decoders d.decoders = make(chan *blockDec, d.o.concurrent) - d.frames = make(chan *frameDec, d.o.concurrent) for i := 0; i < d.o.concurrent; i++ { - d.frames <- newFrameDec(d.o) - d.decoders <- newBlockDec(d.o.lowMem) + dec := newBlockDec(d.o.lowMem) + dec.localFrame = newFrameDec(d.o) + d.decoders <- dec } if r == nil { @@ -283,15 +280,15 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { } // Grab a block decoder and frame decoder. - block, frame := <-d.decoders, <-d.frames + block := <-d.decoders + frame := block.localFrame defer func() { if debug { printf("re-adding decoder: %p", block) } - d.decoders <- block frame.rawInput = nil frame.bBuf = nil - d.frames <- frame + d.decoders <- block }() frame.bBuf = input diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go index 957cfeb7..e6d3d49b 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go @@ -55,7 +55,7 @@ func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { if b.remain() < 4 { return errors.New("input too small") } - bitStream := b.Uint32() + bitStream := b.Uint32NC() nbBits := uint((bitStream & 0xF) + minTablelog) // extract tableLog if nbBits > tablelogAbsoluteMax { println("Invalid tablelog:", nbBits) @@ -79,7 +79,8 @@ func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { n0 += 24 if r := b.remain(); r > 5 { b.advance(2) - bitStream = b.Uint32() >> bitCount + // The check above should make sure we can read 32 bits + bitStream = b.Uint32NC() >> bitCount } else { // end of bit stream bitStream >>= 16 @@ -104,10 +105,11 @@ func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { charnum++ } - if r := b.remain(); r >= 7 || r+int(bitCount>>3) >= 4 { + if r := b.remain(); r >= 7 || r-int(bitCount>>3) >= 4 { b.advance(bitCount >> 3) bitCount &= 7 - bitStream = b.Uint32() >> bitCount + // The check above should make sure we can read 32 bits + bitStream = b.Uint32NC() >> bitCount } else { bitStream >>= 2 } @@ -148,17 +150,16 @@ func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { threshold >>= 1 } - //println("b.off:", b.off, "len:", len(b.b), "bc:", bitCount, "remain:", b.remain()) - if r := b.remain(); r >= 7 || r+int(bitCount>>3) >= 4 { + if r := b.remain(); r >= 7 || r-int(bitCount>>3) >= 4 { b.advance(bitCount >> 3) bitCount &= 7 + // The check above should make sure we can read 32 bits + bitStream = b.Uint32NC() >> (bitCount & 31) } else { bitCount -= (uint)(8 * (len(b.b) - 4 - b.off)) b.off = len(b.b) - 4 - //println("b.off:", b.off, "len:", len(b.b), "bc:", bitCount, "iend", iend) + bitStream = b.Uint32() >> (bitCount & 31) } - bitStream = b.Uint32() >> (bitCount & 31) - //printf("bitstream is now: 0b%b", bitStream) } s.symbolLen = charnum if s.symbolLen <= 1 { diff --git a/vendor/github.com/mattn/go-isatty/.travis.yml b/vendor/github.com/mattn/go-isatty/.travis.yml new file mode 100644 index 00000000..604314dd --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/.travis.yml @@ -0,0 +1,14 @@ +language: go +sudo: false +go: + - 1.13.x + - tip + +before_install: + - go get -t -v ./... + +script: + - ./go.test.sh + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/mattn/go-isatty/LICENSE b/vendor/github.com/mattn/go-isatty/LICENSE new file mode 100644 index 00000000..65dc692b --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) Yasuhiro MATSUMOTO + +MIT License (Expat) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/mattn/go-isatty/README.md b/vendor/github.com/mattn/go-isatty/README.md new file mode 100644 index 00000000..38418353 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/README.md @@ -0,0 +1,50 @@ +# go-isatty + +[![Godoc Reference](https://godoc.org/github.com/mattn/go-isatty?status.svg)](http://godoc.org/github.com/mattn/go-isatty) +[![Codecov](https://codecov.io/gh/mattn/go-isatty/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-isatty) +[![Coverage Status](https://coveralls.io/repos/github/mattn/go-isatty/badge.svg?branch=master)](https://coveralls.io/github/mattn/go-isatty?branch=master) +[![Go Report Card](https://goreportcard.com/badge/mattn/go-isatty)](https://goreportcard.com/report/mattn/go-isatty) + +isatty for golang + +## Usage + +```go +package main + +import ( + "fmt" + "github.com/mattn/go-isatty" + "os" +) + +func main() { + if isatty.IsTerminal(os.Stdout.Fd()) { + fmt.Println("Is Terminal") + } else if isatty.IsCygwinTerminal(os.Stdout.Fd()) { + fmt.Println("Is Cygwin/MSYS2 Terminal") + } else { + fmt.Println("Is Not Terminal") + } +} +``` + +## Installation + +``` +$ go get github.com/mattn/go-isatty +``` + +## License + +MIT + +## Author + +Yasuhiro Matsumoto (a.k.a mattn) + +## Thanks + +* k-takata: base idea for IsCygwinTerminal + + https://github.com/k-takata/go-iscygpty diff --git a/vendor/github.com/mattn/go-isatty/doc.go b/vendor/github.com/mattn/go-isatty/doc.go new file mode 100644 index 00000000..17d4f90e --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/doc.go @@ -0,0 +1,2 @@ +// Package isatty implements interface to isatty +package isatty diff --git a/vendor/github.com/mattn/go-isatty/go.mod b/vendor/github.com/mattn/go-isatty/go.mod new file mode 100644 index 00000000..605c4c22 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/go.mod @@ -0,0 +1,5 @@ +module github.com/mattn/go-isatty + +go 1.12 + +require golang.org/x/sys v0.0.0-20200116001909-b77594299b42 diff --git a/vendor/github.com/mattn/go-isatty/go.sum b/vendor/github.com/mattn/go-isatty/go.sum new file mode 100644 index 00000000..912e29cb --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/mattn/go-isatty/go.test.sh b/vendor/github.com/mattn/go-isatty/go.test.sh new file mode 100644 index 00000000..012162b0 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/go.test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +for d in $(go list ./... | grep -v vendor); do + go test -race -coverprofile=profile.out -covermode=atomic "$d" + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done diff --git a/vendor/github.com/mattn/go-isatty/isatty_bsd.go b/vendor/github.com/mattn/go-isatty/isatty_bsd.go new file mode 100644 index 00000000..711f2880 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_bsd.go @@ -0,0 +1,18 @@ +// +build darwin freebsd openbsd netbsd dragonfly +// +build !appengine + +package isatty + +import "golang.org/x/sys/unix" + +// IsTerminal return true if the file descriptor is terminal. +func IsTerminal(fd uintptr) bool { + _, err := unix.IoctlGetTermios(int(fd), unix.TIOCGETA) + return err == nil +} + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_others.go b/vendor/github.com/mattn/go-isatty/isatty_others.go new file mode 100644 index 00000000..ff714a37 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_others.go @@ -0,0 +1,15 @@ +// +build appengine js nacl + +package isatty + +// IsTerminal returns true if the file descriptor is terminal which +// is always false on js and appengine classic which is a sandboxed PaaS. +func IsTerminal(fd uintptr) bool { + return false +} + +// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_plan9.go b/vendor/github.com/mattn/go-isatty/isatty_plan9.go new file mode 100644 index 00000000..c5b6e0c0 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_plan9.go @@ -0,0 +1,22 @@ +// +build plan9 + +package isatty + +import ( + "syscall" +) + +// IsTerminal returns true if the given file descriptor is a terminal. +func IsTerminal(fd uintptr) bool { + path, err := syscall.Fd2path(int(fd)) + if err != nil { + return false + } + return path == "/dev/cons" || path == "/mnt/term/dev/cons" +} + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_solaris.go b/vendor/github.com/mattn/go-isatty/isatty_solaris.go new file mode 100644 index 00000000..bdd5c79a --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_solaris.go @@ -0,0 +1,22 @@ +// +build solaris +// +build !appengine + +package isatty + +import ( + "golang.org/x/sys/unix" +) + +// IsTerminal returns true if the given file descriptor is a terminal. +// see: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libbc/libc/gen/common/isatty.c +func IsTerminal(fd uintptr) bool { + var termio unix.Termio + err := unix.IoctlSetTermio(int(fd), unix.TCGETA, &termio) + return err == nil +} + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_tcgets.go b/vendor/github.com/mattn/go-isatty/isatty_tcgets.go new file mode 100644 index 00000000..31a1ca97 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_tcgets.go @@ -0,0 +1,18 @@ +// +build linux aix +// +build !appengine + +package isatty + +import "golang.org/x/sys/unix" + +// IsTerminal return true if the file descriptor is terminal. +func IsTerminal(fd uintptr) bool { + _, err := unix.IoctlGetTermios(int(fd), unix.TCGETS) + return err == nil +} + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_windows.go b/vendor/github.com/mattn/go-isatty/isatty_windows.go new file mode 100644 index 00000000..1fa86915 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_windows.go @@ -0,0 +1,125 @@ +// +build windows +// +build !appengine + +package isatty + +import ( + "errors" + "strings" + "syscall" + "unicode/utf16" + "unsafe" +) + +const ( + objectNameInfo uintptr = 1 + fileNameInfo = 2 + fileTypePipe = 3 +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + ntdll = syscall.NewLazyDLL("ntdll.dll") + procGetConsoleMode = kernel32.NewProc("GetConsoleMode") + procGetFileInformationByHandleEx = kernel32.NewProc("GetFileInformationByHandleEx") + procGetFileType = kernel32.NewProc("GetFileType") + procNtQueryObject = ntdll.NewProc("NtQueryObject") +) + +func init() { + // Check if GetFileInformationByHandleEx is available. + if procGetFileInformationByHandleEx.Find() != nil { + procGetFileInformationByHandleEx = nil + } +} + +// IsTerminal return true if the file descriptor is terminal. +func IsTerminal(fd uintptr) bool { + var st uint32 + r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, fd, uintptr(unsafe.Pointer(&st)), 0) + return r != 0 && e == 0 +} + +// Check pipe name is used for cygwin/msys2 pty. +// Cygwin/MSYS2 PTY has a name like: +// \{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master +func isCygwinPipeName(name string) bool { + token := strings.Split(name, "-") + if len(token) < 5 { + return false + } + + if token[0] != `\msys` && + token[0] != `\cygwin` && + token[0] != `\Device\NamedPipe\msys` && + token[0] != `\Device\NamedPipe\cygwin` { + return false + } + + if token[1] == "" { + return false + } + + if !strings.HasPrefix(token[2], "pty") { + return false + } + + if token[3] != `from` && token[3] != `to` { + return false + } + + if token[4] != "master" { + return false + } + + return true +} + +// getFileNameByHandle use the undocomented ntdll NtQueryObject to get file full name from file handler +// since GetFileInformationByHandleEx is not avilable under windows Vista and still some old fashion +// guys are using Windows XP, this is a workaround for those guys, it will also work on system from +// Windows vista to 10 +// see https://stackoverflow.com/a/18792477 for details +func getFileNameByHandle(fd uintptr) (string, error) { + if procNtQueryObject == nil { + return "", errors.New("ntdll.dll: NtQueryObject not supported") + } + + var buf [4 + syscall.MAX_PATH]uint16 + var result int + r, _, e := syscall.Syscall6(procNtQueryObject.Addr(), 5, + fd, objectNameInfo, uintptr(unsafe.Pointer(&buf)), uintptr(2*len(buf)), uintptr(unsafe.Pointer(&result)), 0) + if r != 0 { + return "", e + } + return string(utf16.Decode(buf[4 : 4+buf[0]/2])), nil +} + +// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2 +// terminal. +func IsCygwinTerminal(fd uintptr) bool { + if procGetFileInformationByHandleEx == nil { + name, err := getFileNameByHandle(fd) + if err != nil { + return false + } + return isCygwinPipeName(name) + } + + // Cygwin/msys's pty is a pipe. + ft, _, e := syscall.Syscall(procGetFileType.Addr(), 1, fd, 0, 0) + if ft != fileTypePipe || e != 0 { + return false + } + + var buf [2 + syscall.MAX_PATH]uint16 + r, _, e := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(), + 4, fd, fileNameInfo, uintptr(unsafe.Pointer(&buf)), + uintptr(len(buf)*2), 0, 0) + if r == 0 || e != 0 { + return false + } + + l := *(*uint32)(unsafe.Pointer(&buf)) + return isCygwinPipeName(string(utf16.Decode(buf[2 : 2+l/2]))) +} diff --git a/vendor/github.com/mattn/go-isatty/renovate.json b/vendor/github.com/mattn/go-isatty/renovate.json new file mode 100644 index 00000000..5ae9d96b --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/renovate.json @@ -0,0 +1,8 @@ +{ + "extends": [ + "config:base" + ], + "postUpdateOptions": [ + "gomodTidy" + ] +} diff --git a/vendor/github.com/mattn/go-runewidth/.travis.yml b/vendor/github.com/mattn/go-runewidth/.travis.yml new file mode 100644 index 00000000..6a21813a --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/.travis.yml @@ -0,0 +1,16 @@ +language: go +sudo: false +go: + - 1.13.x + - tip + +before_install: + - go get -t -v ./... + +script: + - go generate + - git diff --cached --exit-code + - ./go.test.sh + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/mattn/go-runewidth/LICENSE b/vendor/github.com/mattn/go-runewidth/LICENSE new file mode 100644 index 00000000..91b5cef3 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/mattn/go-runewidth/README.md b/vendor/github.com/mattn/go-runewidth/README.md new file mode 100644 index 00000000..aa56ab96 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/README.md @@ -0,0 +1,27 @@ +go-runewidth +============ + +[![Build Status](https://travis-ci.org/mattn/go-runewidth.png?branch=master)](https://travis-ci.org/mattn/go-runewidth) +[![Codecov](https://codecov.io/gh/mattn/go-runewidth/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-runewidth) +[![GoDoc](https://godoc.org/github.com/mattn/go-runewidth?status.svg)](http://godoc.org/github.com/mattn/go-runewidth) +[![Go Report Card](https://goreportcard.com/badge/github.com/mattn/go-runewidth)](https://goreportcard.com/report/github.com/mattn/go-runewidth) + +Provides functions to get fixed width of the character or string. + +Usage +----- + +```go +runewidth.StringWidth("つのだ☆HIRO") == 12 +``` + + +Author +------ + +Yasuhiro Matsumoto + +License +------- + +under the MIT License: http://mattn.mit-license.org/2013 diff --git a/vendor/github.com/mattn/go-runewidth/go.mod b/vendor/github.com/mattn/go-runewidth/go.mod new file mode 100644 index 00000000..fa7f4d86 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/go.mod @@ -0,0 +1,3 @@ +module github.com/mattn/go-runewidth + +go 1.9 diff --git a/vendor/github.com/mattn/go-runewidth/go.test.sh b/vendor/github.com/mattn/go-runewidth/go.test.sh new file mode 100644 index 00000000..012162b0 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/go.test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +for d in $(go list ./... | grep -v vendor); do + go test -race -coverprofile=profile.out -covermode=atomic "$d" + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done diff --git a/vendor/github.com/mattn/go-runewidth/runewidth.go b/vendor/github.com/mattn/go-runewidth/runewidth.go new file mode 100644 index 00000000..19f8e044 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth.go @@ -0,0 +1,257 @@ +package runewidth + +import ( + "os" +) + +//go:generate go run script/generate.go + +var ( + // EastAsianWidth will be set true if the current locale is CJK + EastAsianWidth bool + + // ZeroWidthJoiner is flag to set to use UTR#51 ZWJ + ZeroWidthJoiner bool + + // DefaultCondition is a condition in current locale + DefaultCondition = &Condition{} +) + +func init() { + handleEnv() +} + +func handleEnv() { + env := os.Getenv("RUNEWIDTH_EASTASIAN") + if env == "" { + EastAsianWidth = IsEastAsian() + } else { + EastAsianWidth = env == "1" + } + // update DefaultCondition + DefaultCondition.EastAsianWidth = EastAsianWidth + DefaultCondition.ZeroWidthJoiner = ZeroWidthJoiner +} + +type interval struct { + first rune + last rune +} + +type table []interval + +func inTables(r rune, ts ...table) bool { + for _, t := range ts { + if inTable(r, t) { + return true + } + } + return false +} + +func inTable(r rune, t table) bool { + if r < t[0].first { + return false + } + + bot := 0 + top := len(t) - 1 + for top >= bot { + mid := (bot + top) >> 1 + + switch { + case t[mid].last < r: + bot = mid + 1 + case t[mid].first > r: + top = mid - 1 + default: + return true + } + } + + return false +} + +var private = table{ + {0x00E000, 0x00F8FF}, {0x0F0000, 0x0FFFFD}, {0x100000, 0x10FFFD}, +} + +var nonprint = table{ + {0x0000, 0x001F}, {0x007F, 0x009F}, {0x00AD, 0x00AD}, + {0x070F, 0x070F}, {0x180B, 0x180E}, {0x200B, 0x200F}, + {0x2028, 0x202E}, {0x206A, 0x206F}, {0xD800, 0xDFFF}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFE, 0xFFFF}, +} + +// Condition have flag EastAsianWidth whether the current locale is CJK or not. +type Condition struct { + EastAsianWidth bool + ZeroWidthJoiner bool +} + +// NewCondition return new instance of Condition which is current locale. +func NewCondition() *Condition { + return &Condition{ + EastAsianWidth: EastAsianWidth, + ZeroWidthJoiner: ZeroWidthJoiner, + } +} + +// RuneWidth returns the number of cells in r. +// See http://www.unicode.org/reports/tr11/ +func (c *Condition) RuneWidth(r rune) int { + switch { + case r < 0 || r > 0x10FFFF || inTables(r, nonprint, combining, notassigned): + return 0 + case (c.EastAsianWidth && IsAmbiguousWidth(r)) || inTables(r, doublewidth): + return 2 + default: + return 1 + } +} + +func (c *Condition) stringWidth(s string) (width int) { + for _, r := range []rune(s) { + width += c.RuneWidth(r) + } + return width +} + +func (c *Condition) stringWidthZeroJoiner(s string) (width int) { + r1, r2 := rune(0), rune(0) + for _, r := range []rune(s) { + if r == 0xFE0E || r == 0xFE0F { + continue + } + w := c.RuneWidth(r) + if r2 == 0x200D && inTables(r, emoji) && inTables(r1, emoji) { + if width < w { + width = w + } + } else { + width += w + } + r1, r2 = r2, r + } + return width +} + +// StringWidth return width as you can see +func (c *Condition) StringWidth(s string) (width int) { + if c.ZeroWidthJoiner { + return c.stringWidthZeroJoiner(s) + } + return c.stringWidth(s) +} + +// Truncate return string truncated with w cells +func (c *Condition) Truncate(s string, w int, tail string) string { + if c.StringWidth(s) <= w { + return s + } + r := []rune(s) + tw := c.StringWidth(tail) + w -= tw + width := 0 + i := 0 + for ; i < len(r); i++ { + cw := c.RuneWidth(r[i]) + if width+cw > w { + break + } + width += cw + } + return string(r[0:i]) + tail +} + +// Wrap return string wrapped with w cells +func (c *Condition) Wrap(s string, w int) string { + width := 0 + out := "" + for _, r := range []rune(s) { + cw := RuneWidth(r) + if r == '\n' { + out += string(r) + width = 0 + continue + } else if width+cw > w { + out += "\n" + width = 0 + out += string(r) + width += cw + continue + } + out += string(r) + width += cw + } + return out +} + +// FillLeft return string filled in left by spaces in w cells +func (c *Condition) FillLeft(s string, w int) string { + width := c.StringWidth(s) + count := w - width + if count > 0 { + b := make([]byte, count) + for i := range b { + b[i] = ' ' + } + return string(b) + s + } + return s +} + +// FillRight return string filled in left by spaces in w cells +func (c *Condition) FillRight(s string, w int) string { + width := c.StringWidth(s) + count := w - width + if count > 0 { + b := make([]byte, count) + for i := range b { + b[i] = ' ' + } + return s + string(b) + } + return s +} + +// RuneWidth returns the number of cells in r. +// See http://www.unicode.org/reports/tr11/ +func RuneWidth(r rune) int { + return DefaultCondition.RuneWidth(r) +} + +// IsAmbiguousWidth returns whether is ambiguous width or not. +func IsAmbiguousWidth(r rune) bool { + return inTables(r, private, ambiguous) +} + +// IsNeutralWidth returns whether is neutral width or not. +func IsNeutralWidth(r rune) bool { + return inTable(r, neutral) +} + +// StringWidth return width as you can see +func StringWidth(s string) (width int) { + return DefaultCondition.StringWidth(s) +} + +// Truncate return string truncated with w cells +func Truncate(s string, w int, tail string) string { + return DefaultCondition.Truncate(s, w, tail) +} + +// Wrap return string wrapped with w cells +func Wrap(s string, w int) string { + return DefaultCondition.Wrap(s, w) +} + +// FillLeft return string filled in left by spaces in w cells +func FillLeft(s string, w int) string { + return DefaultCondition.FillLeft(s, w) +} + +// FillRight return string filled in left by spaces in w cells +func FillRight(s string, w int) string { + return DefaultCondition.FillRight(s, w) +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go b/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go new file mode 100644 index 00000000..7d99f6e5 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go @@ -0,0 +1,8 @@ +// +build appengine + +package runewidth + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + return false +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_js.go b/vendor/github.com/mattn/go-runewidth/runewidth_js.go new file mode 100644 index 00000000..c5fdf40b --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_js.go @@ -0,0 +1,9 @@ +// +build js +// +build !appengine + +package runewidth + +func IsEastAsian() bool { + // TODO: Implement this for the web. Detect east asian in a compatible way, and return true. + return false +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_posix.go b/vendor/github.com/mattn/go-runewidth/runewidth_posix.go new file mode 100644 index 00000000..480ad748 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_posix.go @@ -0,0 +1,82 @@ +// +build !windows +// +build !js +// +build !appengine + +package runewidth + +import ( + "os" + "regexp" + "strings" +) + +var reLoc = regexp.MustCompile(`^[a-z][a-z][a-z]?(?:_[A-Z][A-Z])?\.(.+)`) + +var mblenTable = map[string]int{ + "utf-8": 6, + "utf8": 6, + "jis": 8, + "eucjp": 3, + "euckr": 2, + "euccn": 2, + "sjis": 2, + "cp932": 2, + "cp51932": 2, + "cp936": 2, + "cp949": 2, + "cp950": 2, + "big5": 2, + "gbk": 2, + "gb2312": 2, +} + +func isEastAsian(locale string) bool { + charset := strings.ToLower(locale) + r := reLoc.FindStringSubmatch(locale) + if len(r) == 2 { + charset = strings.ToLower(r[1]) + } + + if strings.HasSuffix(charset, "@cjk_narrow") { + return false + } + + for pos, b := range []byte(charset) { + if b == '@' { + charset = charset[:pos] + break + } + } + max := 1 + if m, ok := mblenTable[charset]; ok { + max = m + } + if max > 1 && (charset[0] != 'u' || + strings.HasPrefix(locale, "ja") || + strings.HasPrefix(locale, "ko") || + strings.HasPrefix(locale, "zh")) { + return true + } + return false +} + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + locale := os.Getenv("LC_ALL") + if locale == "" { + locale = os.Getenv("LC_CTYPE") + } + if locale == "" { + locale = os.Getenv("LANG") + } + + // ignore C locale + if locale == "POSIX" || locale == "C" { + return false + } + if len(locale) > 1 && locale[0] == 'C' && (locale[1] == '.' || locale[1] == '-') { + return false + } + + return isEastAsian(locale) +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_table.go b/vendor/github.com/mattn/go-runewidth/runewidth_table.go new file mode 100644 index 00000000..b27d77d8 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_table.go @@ -0,0 +1,437 @@ +// Code generated by script/generate.go. DO NOT EDIT. + +package runewidth + +var combining = table{ + {0x0300, 0x036F}, {0x0483, 0x0489}, {0x07EB, 0x07F3}, + {0x0C00, 0x0C00}, {0x0C04, 0x0C04}, {0x0D00, 0x0D01}, + {0x135D, 0x135F}, {0x1A7F, 0x1A7F}, {0x1AB0, 0x1AC0}, + {0x1B6B, 0x1B73}, {0x1DC0, 0x1DF9}, {0x1DFB, 0x1DFF}, + {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2DE0, 0x2DFF}, + {0x3099, 0x309A}, {0xA66F, 0xA672}, {0xA674, 0xA67D}, + {0xA69E, 0xA69F}, {0xA6F0, 0xA6F1}, {0xA8E0, 0xA8F1}, + {0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, {0x10376, 0x1037A}, + {0x10EAB, 0x10EAC}, {0x10F46, 0x10F50}, {0x11300, 0x11301}, + {0x1133B, 0x1133C}, {0x11366, 0x1136C}, {0x11370, 0x11374}, + {0x16AF0, 0x16AF4}, {0x1D165, 0x1D169}, {0x1D16D, 0x1D172}, + {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD}, + {0x1D242, 0x1D244}, {0x1E000, 0x1E006}, {0x1E008, 0x1E018}, + {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, + {0x1E8D0, 0x1E8D6}, +} + +var doublewidth = table{ + {0x1100, 0x115F}, {0x231A, 0x231B}, {0x2329, 0x232A}, + {0x23E9, 0x23EC}, {0x23F0, 0x23F0}, {0x23F3, 0x23F3}, + {0x25FD, 0x25FE}, {0x2614, 0x2615}, {0x2648, 0x2653}, + {0x267F, 0x267F}, {0x2693, 0x2693}, {0x26A1, 0x26A1}, + {0x26AA, 0x26AB}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5}, + {0x26CE, 0x26CE}, {0x26D4, 0x26D4}, {0x26EA, 0x26EA}, + {0x26F2, 0x26F3}, {0x26F5, 0x26F5}, {0x26FA, 0x26FA}, + {0x26FD, 0x26FD}, {0x2705, 0x2705}, {0x270A, 0x270B}, + {0x2728, 0x2728}, {0x274C, 0x274C}, {0x274E, 0x274E}, + {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797}, + {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, {0x2B1B, 0x2B1C}, + {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x2E80, 0x2E99}, + {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, + {0x3000, 0x303E}, {0x3041, 0x3096}, {0x3099, 0x30FF}, + {0x3105, 0x312F}, {0x3131, 0x318E}, {0x3190, 0x31E3}, + {0x31F0, 0x321E}, {0x3220, 0x3247}, {0x3250, 0x4DBF}, + {0x4E00, 0xA48C}, {0xA490, 0xA4C6}, {0xA960, 0xA97C}, + {0xAC00, 0xD7A3}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, + {0xFE30, 0xFE52}, {0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, + {0xFF01, 0xFF60}, {0xFFE0, 0xFFE6}, {0x16FE0, 0x16FE4}, + {0x16FF0, 0x16FF1}, {0x17000, 0x187F7}, {0x18800, 0x18CD5}, + {0x18D00, 0x18D08}, {0x1B000, 0x1B11E}, {0x1B150, 0x1B152}, + {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1F004, 0x1F004}, + {0x1F0CF, 0x1F0CF}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A}, + {0x1F200, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, + {0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F320}, + {0x1F32D, 0x1F335}, {0x1F337, 0x1F37C}, {0x1F37E, 0x1F393}, + {0x1F3A0, 0x1F3CA}, {0x1F3CF, 0x1F3D3}, {0x1F3E0, 0x1F3F0}, + {0x1F3F4, 0x1F3F4}, {0x1F3F8, 0x1F43E}, {0x1F440, 0x1F440}, + {0x1F442, 0x1F4FC}, {0x1F4FF, 0x1F53D}, {0x1F54B, 0x1F54E}, + {0x1F550, 0x1F567}, {0x1F57A, 0x1F57A}, {0x1F595, 0x1F596}, + {0x1F5A4, 0x1F5A4}, {0x1F5FB, 0x1F64F}, {0x1F680, 0x1F6C5}, + {0x1F6CC, 0x1F6CC}, {0x1F6D0, 0x1F6D2}, {0x1F6D5, 0x1F6D7}, + {0x1F6EB, 0x1F6EC}, {0x1F6F4, 0x1F6FC}, {0x1F7E0, 0x1F7EB}, + {0x1F90C, 0x1F93A}, {0x1F93C, 0x1F945}, {0x1F947, 0x1F978}, + {0x1F97A, 0x1F9CB}, {0x1F9CD, 0x1F9FF}, {0x1FA70, 0x1FA74}, + {0x1FA78, 0x1FA7A}, {0x1FA80, 0x1FA86}, {0x1FA90, 0x1FAA8}, + {0x1FAB0, 0x1FAB6}, {0x1FAC0, 0x1FAC2}, {0x1FAD0, 0x1FAD6}, + {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}, +} + +var ambiguous = table{ + {0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8}, + {0x00AA, 0x00AA}, {0x00AD, 0x00AE}, {0x00B0, 0x00B4}, + {0x00B6, 0x00BA}, {0x00BC, 0x00BF}, {0x00C6, 0x00C6}, + {0x00D0, 0x00D0}, {0x00D7, 0x00D8}, {0x00DE, 0x00E1}, + {0x00E6, 0x00E6}, {0x00E8, 0x00EA}, {0x00EC, 0x00ED}, + {0x00F0, 0x00F0}, {0x00F2, 0x00F3}, {0x00F7, 0x00FA}, + {0x00FC, 0x00FC}, {0x00FE, 0x00FE}, {0x0101, 0x0101}, + {0x0111, 0x0111}, {0x0113, 0x0113}, {0x011B, 0x011B}, + {0x0126, 0x0127}, {0x012B, 0x012B}, {0x0131, 0x0133}, + {0x0138, 0x0138}, {0x013F, 0x0142}, {0x0144, 0x0144}, + {0x0148, 0x014B}, {0x014D, 0x014D}, {0x0152, 0x0153}, + {0x0166, 0x0167}, {0x016B, 0x016B}, {0x01CE, 0x01CE}, + {0x01D0, 0x01D0}, {0x01D2, 0x01D2}, {0x01D4, 0x01D4}, + {0x01D6, 0x01D6}, {0x01D8, 0x01D8}, {0x01DA, 0x01DA}, + {0x01DC, 0x01DC}, {0x0251, 0x0251}, {0x0261, 0x0261}, + {0x02C4, 0x02C4}, {0x02C7, 0x02C7}, {0x02C9, 0x02CB}, + {0x02CD, 0x02CD}, {0x02D0, 0x02D0}, {0x02D8, 0x02DB}, + {0x02DD, 0x02DD}, {0x02DF, 0x02DF}, {0x0300, 0x036F}, + {0x0391, 0x03A1}, {0x03A3, 0x03A9}, {0x03B1, 0x03C1}, + {0x03C3, 0x03C9}, {0x0401, 0x0401}, {0x0410, 0x044F}, + {0x0451, 0x0451}, {0x2010, 0x2010}, {0x2013, 0x2016}, + {0x2018, 0x2019}, {0x201C, 0x201D}, {0x2020, 0x2022}, + {0x2024, 0x2027}, {0x2030, 0x2030}, {0x2032, 0x2033}, + {0x2035, 0x2035}, {0x203B, 0x203B}, {0x203E, 0x203E}, + {0x2074, 0x2074}, {0x207F, 0x207F}, {0x2081, 0x2084}, + {0x20AC, 0x20AC}, {0x2103, 0x2103}, {0x2105, 0x2105}, + {0x2109, 0x2109}, {0x2113, 0x2113}, {0x2116, 0x2116}, + {0x2121, 0x2122}, {0x2126, 0x2126}, {0x212B, 0x212B}, + {0x2153, 0x2154}, {0x215B, 0x215E}, {0x2160, 0x216B}, + {0x2170, 0x2179}, {0x2189, 0x2189}, {0x2190, 0x2199}, + {0x21B8, 0x21B9}, {0x21D2, 0x21D2}, {0x21D4, 0x21D4}, + {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203}, + {0x2207, 0x2208}, {0x220B, 0x220B}, {0x220F, 0x220F}, + {0x2211, 0x2211}, {0x2215, 0x2215}, {0x221A, 0x221A}, + {0x221D, 0x2220}, {0x2223, 0x2223}, {0x2225, 0x2225}, + {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237}, + {0x223C, 0x223D}, {0x2248, 0x2248}, {0x224C, 0x224C}, + {0x2252, 0x2252}, {0x2260, 0x2261}, {0x2264, 0x2267}, + {0x226A, 0x226B}, {0x226E, 0x226F}, {0x2282, 0x2283}, + {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299}, + {0x22A5, 0x22A5}, {0x22BF, 0x22BF}, {0x2312, 0x2312}, + {0x2460, 0x24E9}, {0x24EB, 0x254B}, {0x2550, 0x2573}, + {0x2580, 0x258F}, {0x2592, 0x2595}, {0x25A0, 0x25A1}, + {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7}, + {0x25BC, 0x25BD}, {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, + {0x25CB, 0x25CB}, {0x25CE, 0x25D1}, {0x25E2, 0x25E5}, + {0x25EF, 0x25EF}, {0x2605, 0x2606}, {0x2609, 0x2609}, + {0x260E, 0x260F}, {0x261C, 0x261C}, {0x261E, 0x261E}, + {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661}, + {0x2663, 0x2665}, {0x2667, 0x266A}, {0x266C, 0x266D}, + {0x266F, 0x266F}, {0x269E, 0x269F}, {0x26BF, 0x26BF}, + {0x26C6, 0x26CD}, {0x26CF, 0x26D3}, {0x26D5, 0x26E1}, + {0x26E3, 0x26E3}, {0x26E8, 0x26E9}, {0x26EB, 0x26F1}, + {0x26F4, 0x26F4}, {0x26F6, 0x26F9}, {0x26FB, 0x26FC}, + {0x26FE, 0x26FF}, {0x273D, 0x273D}, {0x2776, 0x277F}, + {0x2B56, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF}, + {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, + {0x1F110, 0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F18D}, + {0x1F18F, 0x1F190}, {0x1F19B, 0x1F1AC}, {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}, +} +var notassigned = table{ + {0x27E6, 0x27ED}, {0x2985, 0x2986}, +} + +var neutral = table{ + {0x0000, 0x001F}, {0x007F, 0x00A0}, {0x00A9, 0x00A9}, + {0x00AB, 0x00AB}, {0x00B5, 0x00B5}, {0x00BB, 0x00BB}, + {0x00C0, 0x00C5}, {0x00C7, 0x00CF}, {0x00D1, 0x00D6}, + {0x00D9, 0x00DD}, {0x00E2, 0x00E5}, {0x00E7, 0x00E7}, + {0x00EB, 0x00EB}, {0x00EE, 0x00EF}, {0x00F1, 0x00F1}, + {0x00F4, 0x00F6}, {0x00FB, 0x00FB}, {0x00FD, 0x00FD}, + {0x00FF, 0x0100}, {0x0102, 0x0110}, {0x0112, 0x0112}, + {0x0114, 0x011A}, {0x011C, 0x0125}, {0x0128, 0x012A}, + {0x012C, 0x0130}, {0x0134, 0x0137}, {0x0139, 0x013E}, + {0x0143, 0x0143}, {0x0145, 0x0147}, {0x014C, 0x014C}, + {0x014E, 0x0151}, {0x0154, 0x0165}, {0x0168, 0x016A}, + {0x016C, 0x01CD}, {0x01CF, 0x01CF}, {0x01D1, 0x01D1}, + {0x01D3, 0x01D3}, {0x01D5, 0x01D5}, {0x01D7, 0x01D7}, + {0x01D9, 0x01D9}, {0x01DB, 0x01DB}, {0x01DD, 0x0250}, + {0x0252, 0x0260}, {0x0262, 0x02C3}, {0x02C5, 0x02C6}, + {0x02C8, 0x02C8}, {0x02CC, 0x02CC}, {0x02CE, 0x02CF}, + {0x02D1, 0x02D7}, {0x02DC, 0x02DC}, {0x02DE, 0x02DE}, + {0x02E0, 0x02FF}, {0x0370, 0x0377}, {0x037A, 0x037F}, + {0x0384, 0x038A}, {0x038C, 0x038C}, {0x038E, 0x0390}, + {0x03AA, 0x03B0}, {0x03C2, 0x03C2}, {0x03CA, 0x0400}, + {0x0402, 0x040F}, {0x0450, 0x0450}, {0x0452, 0x052F}, + {0x0531, 0x0556}, {0x0559, 0x058A}, {0x058D, 0x058F}, + {0x0591, 0x05C7}, {0x05D0, 0x05EA}, {0x05EF, 0x05F4}, + {0x0600, 0x061C}, {0x061E, 0x070D}, {0x070F, 0x074A}, + {0x074D, 0x07B1}, {0x07C0, 0x07FA}, {0x07FD, 0x082D}, + {0x0830, 0x083E}, {0x0840, 0x085B}, {0x085E, 0x085E}, + {0x0860, 0x086A}, {0x08A0, 0x08B4}, {0x08B6, 0x08C7}, + {0x08D3, 0x0983}, {0x0985, 0x098C}, {0x098F, 0x0990}, + {0x0993, 0x09A8}, {0x09AA, 0x09B0}, {0x09B2, 0x09B2}, + {0x09B6, 0x09B9}, {0x09BC, 0x09C4}, {0x09C7, 0x09C8}, + {0x09CB, 0x09CE}, {0x09D7, 0x09D7}, {0x09DC, 0x09DD}, + {0x09DF, 0x09E3}, {0x09E6, 0x09FE}, {0x0A01, 0x0A03}, + {0x0A05, 0x0A0A}, {0x0A0F, 0x0A10}, {0x0A13, 0x0A28}, + {0x0A2A, 0x0A30}, {0x0A32, 0x0A33}, {0x0A35, 0x0A36}, + {0x0A38, 0x0A39}, {0x0A3C, 0x0A3C}, {0x0A3E, 0x0A42}, + {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, + {0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E}, {0x0A66, 0x0A76}, + {0x0A81, 0x0A83}, {0x0A85, 0x0A8D}, {0x0A8F, 0x0A91}, + {0x0A93, 0x0AA8}, {0x0AAA, 0x0AB0}, {0x0AB2, 0x0AB3}, + {0x0AB5, 0x0AB9}, {0x0ABC, 0x0AC5}, {0x0AC7, 0x0AC9}, + {0x0ACB, 0x0ACD}, {0x0AD0, 0x0AD0}, {0x0AE0, 0x0AE3}, + {0x0AE6, 0x0AF1}, {0x0AF9, 0x0AFF}, {0x0B01, 0x0B03}, + {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10}, {0x0B13, 0x0B28}, + {0x0B2A, 0x0B30}, {0x0B32, 0x0B33}, {0x0B35, 0x0B39}, + {0x0B3C, 0x0B44}, {0x0B47, 0x0B48}, {0x0B4B, 0x0B4D}, + {0x0B55, 0x0B57}, {0x0B5C, 0x0B5D}, {0x0B5F, 0x0B63}, + {0x0B66, 0x0B77}, {0x0B82, 0x0B83}, {0x0B85, 0x0B8A}, + {0x0B8E, 0x0B90}, {0x0B92, 0x0B95}, {0x0B99, 0x0B9A}, + {0x0B9C, 0x0B9C}, {0x0B9E, 0x0B9F}, {0x0BA3, 0x0BA4}, + {0x0BA8, 0x0BAA}, {0x0BAE, 0x0BB9}, {0x0BBE, 0x0BC2}, + {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCD}, {0x0BD0, 0x0BD0}, + {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BFA}, {0x0C00, 0x0C0C}, + {0x0C0E, 0x0C10}, {0x0C12, 0x0C28}, {0x0C2A, 0x0C39}, + {0x0C3D, 0x0C44}, {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, + {0x0C55, 0x0C56}, {0x0C58, 0x0C5A}, {0x0C60, 0x0C63}, + {0x0C66, 0x0C6F}, {0x0C77, 0x0C8C}, {0x0C8E, 0x0C90}, + {0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9}, + {0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD}, + {0x0CD5, 0x0CD6}, {0x0CDE, 0x0CDE}, {0x0CE0, 0x0CE3}, + {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF2}, {0x0D00, 0x0D0C}, + {0x0D0E, 0x0D10}, {0x0D12, 0x0D44}, {0x0D46, 0x0D48}, + {0x0D4A, 0x0D4F}, {0x0D54, 0x0D63}, {0x0D66, 0x0D7F}, + {0x0D81, 0x0D83}, {0x0D85, 0x0D96}, {0x0D9A, 0x0DB1}, + {0x0DB3, 0x0DBB}, {0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6}, + {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, + {0x0DD8, 0x0DDF}, {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4}, + {0x0E01, 0x0E3A}, {0x0E3F, 0x0E5B}, {0x0E81, 0x0E82}, + {0x0E84, 0x0E84}, {0x0E86, 0x0E8A}, {0x0E8C, 0x0EA3}, + {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EBD}, {0x0EC0, 0x0EC4}, + {0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECD}, {0x0ED0, 0x0ED9}, + {0x0EDC, 0x0EDF}, {0x0F00, 0x0F47}, {0x0F49, 0x0F6C}, + {0x0F71, 0x0F97}, {0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC}, + {0x0FCE, 0x0FDA}, {0x1000, 0x10C5}, {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, {0x10D0, 0x10FF}, {0x1160, 0x1248}, + {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, + {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, + {0x1290, 0x12B0}, {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, + {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, + {0x135D, 0x137C}, {0x1380, 0x1399}, {0x13A0, 0x13F5}, + {0x13F8, 0x13FD}, {0x1400, 0x169C}, {0x16A0, 0x16F8}, + {0x1700, 0x170C}, {0x170E, 0x1714}, {0x1720, 0x1736}, + {0x1740, 0x1753}, {0x1760, 0x176C}, {0x176E, 0x1770}, + {0x1772, 0x1773}, {0x1780, 0x17DD}, {0x17E0, 0x17E9}, + {0x17F0, 0x17F9}, {0x1800, 0x180E}, {0x1810, 0x1819}, + {0x1820, 0x1878}, {0x1880, 0x18AA}, {0x18B0, 0x18F5}, + {0x1900, 0x191E}, {0x1920, 0x192B}, {0x1930, 0x193B}, + {0x1940, 0x1940}, {0x1944, 0x196D}, {0x1970, 0x1974}, + {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, + {0x19DE, 0x1A1B}, {0x1A1E, 0x1A5E}, {0x1A60, 0x1A7C}, + {0x1A7F, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA0, 0x1AAD}, + {0x1AB0, 0x1AC0}, {0x1B00, 0x1B4B}, {0x1B50, 0x1B7C}, + {0x1B80, 0x1BF3}, {0x1BFC, 0x1C37}, {0x1C3B, 0x1C49}, + {0x1C4D, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CC7}, + {0x1CD0, 0x1CFA}, {0x1D00, 0x1DF9}, {0x1DFB, 0x1F15}, + {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, + {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FC4}, {0x1FC6, 0x1FD3}, {0x1FD6, 0x1FDB}, + {0x1FDD, 0x1FEF}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFE}, + {0x2000, 0x200F}, {0x2011, 0x2012}, {0x2017, 0x2017}, + {0x201A, 0x201B}, {0x201E, 0x201F}, {0x2023, 0x2023}, + {0x2028, 0x202F}, {0x2031, 0x2031}, {0x2034, 0x2034}, + {0x2036, 0x203A}, {0x203C, 0x203D}, {0x203F, 0x2064}, + {0x2066, 0x2071}, {0x2075, 0x207E}, {0x2080, 0x2080}, + {0x2085, 0x208E}, {0x2090, 0x209C}, {0x20A0, 0x20A8}, + {0x20AA, 0x20AB}, {0x20AD, 0x20BF}, {0x20D0, 0x20F0}, + {0x2100, 0x2102}, {0x2104, 0x2104}, {0x2106, 0x2108}, + {0x210A, 0x2112}, {0x2114, 0x2115}, {0x2117, 0x2120}, + {0x2123, 0x2125}, {0x2127, 0x212A}, {0x212C, 0x2152}, + {0x2155, 0x215A}, {0x215F, 0x215F}, {0x216C, 0x216F}, + {0x217A, 0x2188}, {0x218A, 0x218B}, {0x219A, 0x21B7}, + {0x21BA, 0x21D1}, {0x21D3, 0x21D3}, {0x21D5, 0x21E6}, + {0x21E8, 0x21FF}, {0x2201, 0x2201}, {0x2204, 0x2206}, + {0x2209, 0x220A}, {0x220C, 0x220E}, {0x2210, 0x2210}, + {0x2212, 0x2214}, {0x2216, 0x2219}, {0x221B, 0x221C}, + {0x2221, 0x2222}, {0x2224, 0x2224}, {0x2226, 0x2226}, + {0x222D, 0x222D}, {0x222F, 0x2233}, {0x2238, 0x223B}, + {0x223E, 0x2247}, {0x2249, 0x224B}, {0x224D, 0x2251}, + {0x2253, 0x225F}, {0x2262, 0x2263}, {0x2268, 0x2269}, + {0x226C, 0x226D}, {0x2270, 0x2281}, {0x2284, 0x2285}, + {0x2288, 0x2294}, {0x2296, 0x2298}, {0x229A, 0x22A4}, + {0x22A6, 0x22BE}, {0x22C0, 0x2311}, {0x2313, 0x2319}, + {0x231C, 0x2328}, {0x232B, 0x23E8}, {0x23ED, 0x23EF}, + {0x23F1, 0x23F2}, {0x23F4, 0x2426}, {0x2440, 0x244A}, + {0x24EA, 0x24EA}, {0x254C, 0x254F}, {0x2574, 0x257F}, + {0x2590, 0x2591}, {0x2596, 0x259F}, {0x25A2, 0x25A2}, + {0x25AA, 0x25B1}, {0x25B4, 0x25B5}, {0x25B8, 0x25BB}, + {0x25BE, 0x25BF}, {0x25C2, 0x25C5}, {0x25C9, 0x25CA}, + {0x25CC, 0x25CD}, {0x25D2, 0x25E1}, {0x25E6, 0x25EE}, + {0x25F0, 0x25FC}, {0x25FF, 0x2604}, {0x2607, 0x2608}, + {0x260A, 0x260D}, {0x2610, 0x2613}, {0x2616, 0x261B}, + {0x261D, 0x261D}, {0x261F, 0x263F}, {0x2641, 0x2641}, + {0x2643, 0x2647}, {0x2654, 0x265F}, {0x2662, 0x2662}, + {0x2666, 0x2666}, {0x266B, 0x266B}, {0x266E, 0x266E}, + {0x2670, 0x267E}, {0x2680, 0x2692}, {0x2694, 0x269D}, + {0x26A0, 0x26A0}, {0x26A2, 0x26A9}, {0x26AC, 0x26BC}, + {0x26C0, 0x26C3}, {0x26E2, 0x26E2}, {0x26E4, 0x26E7}, + {0x2700, 0x2704}, {0x2706, 0x2709}, {0x270C, 0x2727}, + {0x2729, 0x273C}, {0x273E, 0x274B}, {0x274D, 0x274D}, + {0x274F, 0x2752}, {0x2756, 0x2756}, {0x2758, 0x2775}, + {0x2780, 0x2794}, {0x2798, 0x27AF}, {0x27B1, 0x27BE}, + {0x27C0, 0x27E5}, {0x27EE, 0x2984}, {0x2987, 0x2B1A}, + {0x2B1D, 0x2B4F}, {0x2B51, 0x2B54}, {0x2B5A, 0x2B73}, + {0x2B76, 0x2B95}, {0x2B97, 0x2C2E}, {0x2C30, 0x2C5E}, + {0x2C60, 0x2CF3}, {0x2CF9, 0x2D25}, {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D70}, + {0x2D7F, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, + {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, + {0x2DE0, 0x2E52}, {0x303F, 0x303F}, {0x4DC0, 0x4DFF}, + {0xA4D0, 0xA62B}, {0xA640, 0xA6F7}, {0xA700, 0xA7BF}, + {0xA7C2, 0xA7CA}, {0xA7F5, 0xA82C}, {0xA830, 0xA839}, + {0xA840, 0xA877}, {0xA880, 0xA8C5}, {0xA8CE, 0xA8D9}, + {0xA8E0, 0xA953}, {0xA95F, 0xA95F}, {0xA980, 0xA9CD}, + {0xA9CF, 0xA9D9}, {0xA9DE, 0xA9FE}, {0xAA00, 0xAA36}, + {0xAA40, 0xAA4D}, {0xAA50, 0xAA59}, {0xAA5C, 0xAAC2}, + {0xAADB, 0xAAF6}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, + {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, + {0xAB30, 0xAB6B}, {0xAB70, 0xABED}, {0xABF0, 0xABF9}, + {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xD800, 0xDFFF}, + {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB36}, + {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, + {0xFB43, 0xFB44}, {0xFB46, 0xFBC1}, {0xFBD3, 0xFD3F}, + {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFD}, + {0xFE20, 0xFE2F}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFC}, {0x10000, 0x1000B}, + {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, + {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, + {0x10100, 0x10102}, {0x10107, 0x10133}, {0x10137, 0x1018E}, + {0x10190, 0x1019C}, {0x101A0, 0x101A0}, {0x101D0, 0x101FD}, + {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E0, 0x102FB}, + {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x1037A}, + {0x10380, 0x1039D}, {0x1039F, 0x103C3}, {0x103C8, 0x103D5}, + {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, + {0x1056F, 0x1056F}, {0x10600, 0x10736}, {0x10740, 0x10755}, + {0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808}, + {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, + {0x1083F, 0x10855}, {0x10857, 0x1089E}, {0x108A7, 0x108AF}, + {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, + {0x1091F, 0x10939}, {0x1093F, 0x1093F}, {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, {0x109D2, 0x10A03}, {0x10A05, 0x10A06}, + {0x10A0C, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, + {0x10A38, 0x10A3A}, {0x10A3F, 0x10A48}, {0x10A50, 0x10A58}, + {0x10A60, 0x10A9F}, {0x10AC0, 0x10AE6}, {0x10AEB, 0x10AF6}, + {0x10B00, 0x10B35}, {0x10B39, 0x10B55}, {0x10B58, 0x10B72}, + {0x10B78, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, + {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10D27}, {0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, + {0x10E80, 0x10EA9}, {0x10EAB, 0x10EAD}, {0x10EB0, 0x10EB1}, + {0x10F00, 0x10F27}, {0x10F30, 0x10F59}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11000, 0x1104D}, {0x11052, 0x1106F}, + {0x1107F, 0x110C1}, {0x110CD, 0x110CD}, {0x110D0, 0x110E8}, + {0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11147}, + {0x11150, 0x11176}, {0x11180, 0x111DF}, {0x111E1, 0x111F4}, + {0x11200, 0x11211}, {0x11213, 0x1123E}, {0x11280, 0x11286}, + {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, + {0x1129F, 0x112A9}, {0x112B0, 0x112EA}, {0x112F0, 0x112F9}, + {0x11300, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, + {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, + {0x11335, 0x11339}, {0x1133B, 0x11344}, {0x11347, 0x11348}, + {0x1134B, 0x1134D}, {0x11350, 0x11350}, {0x11357, 0x11357}, + {0x1135D, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, + {0x11400, 0x1145B}, {0x1145D, 0x11461}, {0x11480, 0x114C7}, + {0x114D0, 0x114D9}, {0x11580, 0x115B5}, {0x115B8, 0x115DD}, + {0x11600, 0x11644}, {0x11650, 0x11659}, {0x11660, 0x1166C}, + {0x11680, 0x116B8}, {0x116C0, 0x116C9}, {0x11700, 0x1171A}, + {0x1171D, 0x1172B}, {0x11730, 0x1173F}, {0x11800, 0x1183B}, + {0x118A0, 0x118F2}, {0x118FF, 0x11906}, {0x11909, 0x11909}, + {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x11935}, + {0x11937, 0x11938}, {0x1193B, 0x11946}, {0x11950, 0x11959}, + {0x119A0, 0x119A7}, {0x119AA, 0x119D7}, {0x119DA, 0x119E4}, + {0x11A00, 0x11A47}, {0x11A50, 0x11AA2}, {0x11AC0, 0x11AF8}, + {0x11C00, 0x11C08}, {0x11C0A, 0x11C36}, {0x11C38, 0x11C45}, + {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F}, {0x11C92, 0x11CA7}, + {0x11CA9, 0x11CB6}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, + {0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, + {0x11D3F, 0x11D47}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, + {0x11D67, 0x11D68}, {0x11D6A, 0x11D8E}, {0x11D90, 0x11D91}, + {0x11D93, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF8}, + {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FF1}, {0x11FFF, 0x12399}, + {0x12400, 0x1246E}, {0x12470, 0x12474}, {0x12480, 0x12543}, + {0x13000, 0x1342E}, {0x13430, 0x13438}, {0x14400, 0x14646}, + {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, + {0x16A6E, 0x16A6F}, {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5}, + {0x16B00, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, + {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E9A}, + {0x16F00, 0x16F4A}, {0x16F4F, 0x16F87}, {0x16F8F, 0x16F9F}, + {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, + {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BCA3}, {0x1D000, 0x1D0F5}, + {0x1D100, 0x1D126}, {0x1D129, 0x1D1E8}, {0x1D200, 0x1D245}, + {0x1D2E0, 0x1D2F3}, {0x1D300, 0x1D356}, {0x1D360, 0x1D378}, + {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, + {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, + {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, + {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, + {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, + {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D7CB}, {0x1D7CE, 0x1DA8B}, + {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, + {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, {0x1E100, 0x1E12C}, {0x1E130, 0x1E13D}, + {0x1E140, 0x1E149}, {0x1E14E, 0x1E14F}, {0x1E2C0, 0x1E2F9}, + {0x1E2FF, 0x1E2FF}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6}, + {0x1E900, 0x1E94B}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F}, + {0x1EC71, 0x1ECB4}, {0x1ED01, 0x1ED3D}, {0x1EE00, 0x1EE03}, + {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, + {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, + {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, + {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, + {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, + {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, + {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, + {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, + {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, + {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1EEF0, 0x1EEF1}, + {0x1F000, 0x1F003}, {0x1F005, 0x1F02B}, {0x1F030, 0x1F093}, + {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, {0x1F0C1, 0x1F0CE}, + {0x1F0D1, 0x1F0F5}, {0x1F10B, 0x1F10F}, {0x1F12E, 0x1F12F}, + {0x1F16A, 0x1F16F}, {0x1F1AD, 0x1F1AD}, {0x1F1E6, 0x1F1FF}, + {0x1F321, 0x1F32C}, {0x1F336, 0x1F336}, {0x1F37D, 0x1F37D}, + {0x1F394, 0x1F39F}, {0x1F3CB, 0x1F3CE}, {0x1F3D4, 0x1F3DF}, + {0x1F3F1, 0x1F3F3}, {0x1F3F5, 0x1F3F7}, {0x1F43F, 0x1F43F}, + {0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FE}, {0x1F53E, 0x1F54A}, + {0x1F54F, 0x1F54F}, {0x1F568, 0x1F579}, {0x1F57B, 0x1F594}, + {0x1F597, 0x1F5A3}, {0x1F5A5, 0x1F5FA}, {0x1F650, 0x1F67F}, + {0x1F6C6, 0x1F6CB}, {0x1F6CD, 0x1F6CF}, {0x1F6D3, 0x1F6D4}, + {0x1F6E0, 0x1F6EA}, {0x1F6F0, 0x1F6F3}, {0x1F700, 0x1F773}, + {0x1F780, 0x1F7D8}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, + {0x1F850, 0x1F859}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, + {0x1F8B0, 0x1F8B1}, {0x1F900, 0x1F90B}, {0x1F93B, 0x1F93B}, + {0x1F946, 0x1F946}, {0x1FA00, 0x1FA53}, {0x1FA60, 0x1FA6D}, + {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA}, {0x1FBF0, 0x1FBF9}, + {0xE0001, 0xE0001}, {0xE0020, 0xE007F}, +} + +var emoji = table{ + {0x203C, 0x203C}, {0x2049, 0x2049}, {0x2122, 0x2122}, + {0x2139, 0x2139}, {0x2194, 0x2199}, {0x21A9, 0x21AA}, + {0x231A, 0x231B}, {0x2328, 0x2328}, {0x2388, 0x2388}, + {0x23CF, 0x23CF}, {0x23E9, 0x23F3}, {0x23F8, 0x23FA}, + {0x24C2, 0x24C2}, {0x25AA, 0x25AB}, {0x25B6, 0x25B6}, + {0x25C0, 0x25C0}, {0x25FB, 0x25FE}, {0x2600, 0x2605}, + {0x2607, 0x2612}, {0x2614, 0x2685}, {0x2690, 0x2705}, + {0x2708, 0x2712}, {0x2714, 0x2714}, {0x2716, 0x2716}, + {0x271D, 0x271D}, {0x2721, 0x2721}, {0x2728, 0x2728}, + {0x2733, 0x2734}, {0x2744, 0x2744}, {0x2747, 0x2747}, + {0x274C, 0x274C}, {0x274E, 0x274E}, {0x2753, 0x2755}, + {0x2757, 0x2757}, {0x2763, 0x2767}, {0x2795, 0x2797}, + {0x27A1, 0x27A1}, {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, + {0x2934, 0x2935}, {0x2B05, 0x2B07}, {0x2B1B, 0x2B1C}, + {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x3030, 0x3030}, + {0x303D, 0x303D}, {0x3297, 0x3297}, {0x3299, 0x3299}, + {0x1F000, 0x1F0FF}, {0x1F10D, 0x1F10F}, {0x1F12F, 0x1F12F}, + {0x1F16C, 0x1F171}, {0x1F17E, 0x1F17F}, {0x1F18E, 0x1F18E}, + {0x1F191, 0x1F19A}, {0x1F1AD, 0x1F1E5}, {0x1F201, 0x1F20F}, + {0x1F21A, 0x1F21A}, {0x1F22F, 0x1F22F}, {0x1F232, 0x1F23A}, + {0x1F23C, 0x1F23F}, {0x1F249, 0x1F3FA}, {0x1F400, 0x1F53D}, + {0x1F546, 0x1F64F}, {0x1F680, 0x1F6FF}, {0x1F774, 0x1F77F}, + {0x1F7D5, 0x1F7FF}, {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F}, + {0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F8FF}, + {0x1F90C, 0x1F93A}, {0x1F93C, 0x1F945}, {0x1F947, 0x1FAFF}, + {0x1FC00, 0x1FFFD}, +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_windows.go b/vendor/github.com/mattn/go-runewidth/runewidth_windows.go new file mode 100644 index 00000000..d6a61777 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_windows.go @@ -0,0 +1,28 @@ +// +build windows +// +build !appengine + +package runewidth + +import ( + "syscall" +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32") + procGetConsoleOutputCP = kernel32.NewProc("GetConsoleOutputCP") +) + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + r1, _, _ := procGetConsoleOutputCP.Call() + if r1 == 0 { + return false + } + + switch int(r1) { + case 932, 51932, 936, 949, 950: + return true + } + + return false +} diff --git a/vendor/github.com/vbauerster/mpb/v5/bar.go b/vendor/github.com/vbauerster/mpb/v5/bar.go index 13bda224..9c28a07a 100644 --- a/vendor/github.com/vbauerster/mpb/v5/bar.go +++ b/vendor/github.com/vbauerster/mpb/v5/bar.go @@ -6,28 +6,15 @@ import ( "fmt" "io" "log" + "runtime/debug" "strings" "time" - "unicode/utf8" + "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" "github.com/vbauerster/mpb/v5/decor" ) -// BarFiller interface. -// Bar renders itself by calling BarFiller's Fill method. You can -// literally have any bar kind, by implementing this interface and -// passing it to the *Progress.Add(...) *Bar method. -type BarFiller interface { - Fill(w io.Writer, width int, stat *decor.Statistics) -} - -// BarFillerFunc is function type adapter to convert function into Filler. -type BarFillerFunc func(w io.Writer, width int, stat *decor.Statistics) - -func (f BarFillerFunc) Fill(w io.Writer, width int, stat *decor.Statistics) { - f(w, width, stat) -} - // Bar represents a progress Bar. type Bar struct { priority int // used by heap @@ -55,21 +42,22 @@ type Bar struct { recoveredPanic interface{} } -type extFunc func(in io.Reader, tw int, st *decor.Statistics) (out io.Reader, lines int) +type extFunc func(in io.Reader, reqWidth int, st decor.Statistics) (out io.Reader, lines int) type bState struct { - baseF BarFiller - filler BarFiller id int - width int + priority int + reqWidth int total int64 current int64 + refill int64 lastN int64 iterated bool trimSpace bool toComplete bool completeFlushed bool ignoreComplete bool + dropOnComplete bool noPop bool aDecorators []decor.Decorator pDecorators []decor.Decorator @@ -77,12 +65,10 @@ type bState struct { ewmaDecorators []decor.EwmaDecorator shutdownListeners []decor.ShutdownListener bufP, bufB, bufA *bytes.Buffer + filler BarFiller + middleware func(BarFiller) BarFiller extender extFunc - // priority overrides *Bar's priority, if set - priority int - // dropOnComplete propagates to *Bar - dropOnComplete bool // runningBar is a key for *pState.parkedBars runningBar *Bar @@ -146,13 +132,8 @@ func (b *Bar) Current() int64 { // Given default bar style is "[=>-]<+", refill rune is '+'. // To set bar style use mpb.BarStyle(string) BarOption. func (b *Bar) SetRefill(amount int64) { - type refiller interface { - SetRefill(int64) - } b.operateState <- func(s *bState) { - if f, ok := s.baseF.(refiller); ok { - f.SetRefill(amount) - } + s.refill = amount } } @@ -318,44 +299,40 @@ func (b *Bar) serve(ctx context.Context, s *bState) { } func (b *Bar) render(tw int) { - if b.recoveredPanic != nil { - b.toShutdown = false - b.frameCh <- b.panicToFrame(tw) - return - } select { case b.operateState <- func(s *bState) { + stat := newStatistics(tw, s) defer func() { // recovering if user defined decorator panics for example if p := recover(); p != nil { - b.dlogger.Println(p) + s.extender = makePanicExtender(p) + frame, lines := s.extender(nil, s.reqWidth, stat) + b.extendedLines = lines + b.toShutdown = !b.toShutdown b.recoveredPanic = p - b.toShutdown = !s.completeFlushed - b.frameCh <- b.panicToFrame(tw) + b.frameCh <- frame + b.dlogger.Println(p) } + s.completeFlushed = s.toComplete }() - - st := newStatistics(s) - frame := s.draw(tw, st) - frame, b.extendedLines = s.extender(frame, tw, st) - + frame, lines := s.extender(s.draw(stat), s.reqWidth, stat) + b.extendedLines = lines b.toShutdown = s.toComplete && !s.completeFlushed - s.completeFlushed = s.toComplete b.frameCh <- frame }: case <-b.done: s := b.cacheState - st := newStatistics(s) - frame := s.draw(tw, st) - frame, b.extendedLines = s.extender(frame, tw, st) + stat := newStatistics(tw, s) + var r io.Reader + if b.recoveredPanic == nil { + r = s.draw(stat) + } + frame, lines := s.extender(r, s.reqWidth, stat) + b.extendedLines = lines b.frameCh <- frame } } -func (b *Bar) panicToFrame(termWidth int) io.Reader { - return strings.NewReader(fmt.Sprintf(fmt.Sprintf("%%.%dv\n", termWidth), b.recoveredPanic)) -} - func (b *Bar) subscribeDecorators() { var averageDecorators []decor.AverageDecorator var ewmaDecorators []decor.EwmaDecorator @@ -398,34 +375,41 @@ func (b *Bar) wSyncTable() [][]chan int { } } -func (s *bState) draw(termWidth int, stat *decor.Statistics) io.Reader { +func (s *bState) draw(stat decor.Statistics) io.Reader { + if !s.trimSpace { + stat.AvailableWidth -= 2 + s.bufB.WriteByte(' ') + defer s.bufB.WriteByte(' ') + } + + nlr := strings.NewReader("\n") + tw := stat.AvailableWidth for _, d := range s.pDecorators { - s.bufP.WriteString(d.Decor(stat)) + str := d.Decor(stat) + stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str)) + s.bufP.WriteString(str) + } + if stat.AvailableWidth <= 0 { + trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufP.String()), tw, "…")) + s.bufP.Reset() + return io.MultiReader(trunc, s.bufB, nlr) } + tw = stat.AvailableWidth for _, d := range s.aDecorators { - s.bufA.WriteString(d.Decor(stat)) + str := d.Decor(stat) + stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str)) + s.bufA.WriteString(str) + } + if stat.AvailableWidth <= 0 { + trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufA.String()), tw, "…")) + s.bufA.Reset() + return io.MultiReader(s.bufP, s.bufB, trunc, nlr) } - s.bufA.WriteByte('\n') + s.filler.Fill(s.bufB, s.reqWidth, stat) - prependCount := utf8.RuneCount(s.bufP.Bytes()) - appendCount := utf8.RuneCount(s.bufA.Bytes()) - 1 - - if fitWidth := s.width; termWidth > 1 { - if !s.trimSpace { - // reserve space for edge spaces - termWidth -= 2 - s.bufB.WriteByte(' ') - defer s.bufB.WriteByte(' ') - } - if prependCount+s.width+appendCount > termWidth { - fitWidth = termWidth - prependCount - appendCount - } - s.filler.Fill(s.bufB, fitWidth, stat) - } - - return io.MultiReader(s.bufP, s.bufB, s.bufA) + return io.MultiReader(s.bufP, s.bufB, s.bufA, nlr) } func (s *bState) wSyncTable() [][]chan int { @@ -450,12 +434,14 @@ func (s *bState) wSyncTable() [][]chan int { return table } -func newStatistics(s *bState) *decor.Statistics { - return &decor.Statistics{ - ID: s.id, - Completed: s.completeFlushed, - Total: s.total, - Current: s.current, +func newStatistics(tw int, s *bState) decor.Statistics { + return decor.Statistics{ + ID: s.id, + AvailableWidth: tw, + Total: s.total, + Current: s.current, + Refill: s.refill, + Completed: s.completeFlushed, } } @@ -476,3 +462,17 @@ func ewmaIterationUpdate(done bool, s *bState, dur time.Duration) { d.EwmaUpdate(s.lastN, dur) } } + +func makePanicExtender(p interface{}) extFunc { + pstr := fmt.Sprint(p) + stack := debug.Stack() + stackLines := bytes.Count(stack, []byte("\n")) + return func(_ io.Reader, _ int, st decor.Statistics) (io.Reader, int) { + mr := io.MultiReader( + strings.NewReader(runewidth.Truncate(pstr, st.AvailableWidth, "…")), + strings.NewReader(fmt.Sprintf("\n%#v\n", st)), + bytes.NewReader(stack), + ) + return mr, stackLines + 1 + } +} diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler.go index 33dbf191..07148bff 100644 --- a/vendor/github.com/vbauerster/mpb/v5/bar_filler.go +++ b/vendor/github.com/vbauerster/mpb/v5/bar_filler.go @@ -2,137 +2,29 @@ package mpb import ( "io" - "unicode/utf8" "github.com/vbauerster/mpb/v5/decor" - "github.com/vbauerster/mpb/v5/internal" ) -const ( - rLeft = iota - rFill - rTip - rEmpty - rRight - rRevTip - rRefill -) - -// DefaultBarStyle is a string containing 7 runes. -// Each rune is a building block of a progress bar. +// BarFiller interface. +// Bar (without decorators) renders itself by calling BarFiller's Fill method. // -// '1st rune' stands for left boundary rune +// `reqWidth` is requested width, which is set via: +// func WithWidth(width int) ContainerOption +// func BarWidth(width int) BarOption // -// '2nd rune' stands for fill rune +// Default implementations can be obtained via: // -// '3rd rune' stands for tip rune +// func NewBarFiller(style string, reverse bool) BarFiller +// func NewSpinnerFiller(style []string, alignment SpinnerAlignment) BarFiller // -// '4th rune' stands for empty rune -// -// '5th rune' stands for right boundary rune -// -// '6th rune' stands for reverse tip rune -// -// '7th rune' stands for refill rune -// -const DefaultBarStyle string = "[=>-]<+" - -type barFiller struct { - format [][]byte - tip []byte - refill int64 - reverse bool - flush func(w io.Writer, bb [][]byte) +type BarFiller interface { + Fill(w io.Writer, reqWidth int, stat decor.Statistics) } -// NewBarFiller constucts mpb.BarFiller, to be used with *Progress.Add(...) *Bar method. -func NewBarFiller(style string, reverse bool) BarFiller { - if style == "" { - style = DefaultBarStyle - } - bf := &barFiller{ - format: make([][]byte, utf8.RuneCountInString(style)), - reverse: reverse, - } - bf.SetStyle(style) - return bf -} - -func (s *barFiller) SetStyle(style string) { - if !utf8.ValidString(style) { - return - } - src := make([][]byte, 0, utf8.RuneCountInString(style)) - for _, r := range style { - src = append(src, []byte(string(r))) - } - copy(s.format, src) - s.SetReverse(s.reverse) -} - -func (s *barFiller) SetReverse(reverse bool) { - if reverse { - s.tip = s.format[rRevTip] - s.flush = reverseFlush - } else { - s.tip = s.format[rTip] - s.flush = regularFlush - } - s.reverse = reverse -} - -func (s *barFiller) SetRefill(amount int64) { - s.refill = amount -} - -func (s *barFiller) Fill(w io.Writer, width int, stat *decor.Statistics) { - // don't count rLeft and rRight as progress - width -= 2 - if width < 2 { - return - } - w.Write(s.format[rLeft]) - defer w.Write(s.format[rRight]) - - bb := make([][]byte, width) - - cwidth := int(internal.PercentageRound(stat.Total, stat.Current, width)) - - for i := 0; i < cwidth; i++ { - bb[i] = s.format[rFill] - } - - if s.refill > 0 { - var rwidth int - if s.refill > stat.Current { - rwidth = cwidth - } else { - rwidth = int(internal.PercentageRound(stat.Total, int64(s.refill), width)) - } - for i := 0; i < rwidth; i++ { - bb[i] = s.format[rRefill] - } - } - - if cwidth > 0 && cwidth < width { - bb[cwidth-1] = s.tip - } - - for i := cwidth; i < width; i++ { - bb[i] = s.format[rEmpty] - } - - s.flush(w, bb) -} - -func regularFlush(w io.Writer, bb [][]byte) { - for i := 0; i < len(bb); i++ { - w.Write(bb[i]) - } -} - -func reverseFlush(w io.Writer, bb [][]byte) { - for i := len(bb) - 1; i >= 0; i-- { - w.Write(bb[i]) - } +// BarFillerFunc is function type adapter to convert function into BarFiller. +type BarFillerFunc func(w io.Writer, reqWidth int, stat decor.Statistics) + +func (f BarFillerFunc) Fill(w io.Writer, reqWidth int, stat decor.Statistics) { + f(w, reqWidth, stat) } diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go new file mode 100644 index 00000000..637bd88c --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go @@ -0,0 +1,173 @@ +package mpb + +import ( + "bytes" + "io" + "unicode/utf8" + + "github.com/mattn/go-runewidth" + "github.com/vbauerster/mpb/v5/decor" + "github.com/vbauerster/mpb/v5/internal" +) + +const ( + rLeft = iota + rFill + rTip + rSpace + rRight + rRevTip + rRefill +) + +// DefaultBarStyle is a string containing 7 runes. +// Each rune is a building block of a progress bar. +// +// '1st rune' stands for left boundary rune +// +// '2nd rune' stands for fill rune +// +// '3rd rune' stands for tip rune +// +// '4th rune' stands for space rune +// +// '5th rune' stands for right boundary rune +// +// '6th rune' stands for reverse tip rune +// +// '7th rune' stands for refill rune +// +const DefaultBarStyle string = "[=>-]<+" + +type barFiller struct { + format [][]byte + rwidth []int + tip []byte + refill int64 + reverse bool + flush func(io.Writer, *space, [][]byte) +} + +type space struct { + space []byte + rwidth int + count int +} + +// NewBarFiller constucts mpb.BarFiller, to be used with *Progress.Add(...) *Bar method. +func NewBarFiller(style string, reverse bool) BarFiller { + bf := &barFiller{ + format: make([][]byte, len(DefaultBarStyle)), + rwidth: make([]int, len(DefaultBarStyle)), + reverse: reverse, + } + bf.SetStyle(style) + return bf +} + +func (s *barFiller) SetStyle(style string) { + if !utf8.ValidString(style) { + panic("invalid bar style") + } + if style == "" { + style = DefaultBarStyle + } + src := make([][]byte, utf8.RuneCountInString(style)) + i := 0 + for _, r := range style { + s.rwidth[i] = runewidth.RuneWidth(r) + src[i] = []byte(string(r)) + i++ + } + copy(s.format, src) + s.SetReverse(s.reverse) +} + +func (s *barFiller) SetReverse(reverse bool) { + if reverse { + s.tip = s.format[rRevTip] + s.flush = reverseFlush + } else { + s.tip = s.format[rTip] + s.flush = regularFlush + } + s.reverse = reverse +} + +func (s *barFiller) Fill(w io.Writer, reqWidth int, stat decor.Statistics) { + width := internal.WidthForBarFiller(reqWidth, stat.AvailableWidth) + + if brackets := s.rwidth[rLeft] + s.rwidth[rRight]; width < brackets { + return + } else { + // don't count brackets as progress + width -= brackets + } + w.Write(s.format[rLeft]) + defer w.Write(s.format[rRight]) + + cwidth := int(internal.PercentageRound(stat.Total, stat.Current, width)) + space := &space{ + space: s.format[rSpace], + rwidth: s.rwidth[rSpace], + count: width - cwidth, + } + + index, refill := 0, 0 + bb := make([][]byte, cwidth) + + if cwidth > 0 && cwidth != width { + bb[index] = s.tip + cwidth -= s.rwidth[rTip] + index++ + } + + if stat.Refill > 0 { + refill = int(internal.PercentageRound(stat.Total, int64(stat.Refill), width)) + if refill > cwidth { + refill = cwidth + } + cwidth -= refill + } + + for cwidth > 0 { + bb[index] = s.format[rFill] + cwidth -= s.rwidth[rFill] + index++ + } + + for refill > 0 { + bb[index] = s.format[rRefill] + refill -= s.rwidth[rRefill] + index++ + } + + if cwidth+refill < 0 || space.rwidth > 1 { + buf := new(bytes.Buffer) + s.flush(buf, space, bb[:index]) + io.WriteString(w, runewidth.Truncate(buf.String(), width, "…")) + return + } + + s.flush(w, space, bb) +} + +func regularFlush(w io.Writer, space *space, bb [][]byte) { + for i := len(bb) - 1; i >= 0; i-- { + w.Write(bb[i]) + } + for space.count > 0 { + w.Write(space.space) + space.count -= space.rwidth + } +} + +func reverseFlush(w io.Writer, space *space, bb [][]byte) { + for space.count > 0 { + w.Write(space.space) + space.count -= space.rwidth + } + for i := 0; i < len(bb); i++ { + w.Write(bb[i]) + } +} diff --git a/vendor/github.com/vbauerster/mpb/v5/spinner_filler.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go similarity index 87% rename from vendor/github.com/vbauerster/mpb/v5/spinner_filler.go rename to vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go index 517725fb..d2cb2b72 100644 --- a/vendor/github.com/vbauerster/mpb/v5/spinner_filler.go +++ b/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go @@ -6,6 +6,7 @@ import ( "unicode/utf8" "github.com/vbauerster/mpb/v5/decor" + "github.com/vbauerster/mpb/v5/internal" ) // SpinnerAlignment enum. @@ -39,7 +40,8 @@ func NewSpinnerFiller(style []string, alignment SpinnerAlignment) BarFiller { return filler } -func (s *spinnerFiller) Fill(w io.Writer, width int, stat *decor.Statistics) { +func (s *spinnerFiller) Fill(w io.Writer, reqWidth int, stat decor.Statistics) { + width := internal.WidthForBarFiller(reqWidth, stat.AvailableWidth) frame := s.frames[s.count%uint(len(s.frames))] frameWidth := utf8.RuneCountInString(frame) diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_option.go b/vendor/github.com/vbauerster/mpb/v5/bar_option.go index 76f2050f..31b7939b 100644 --- a/vendor/github.com/vbauerster/mpb/v5/bar_option.go +++ b/vendor/github.com/vbauerster/mpb/v5/bar_option.go @@ -46,7 +46,7 @@ func BarID(id int) BarOption { // BarWidth sets bar width independent of the container. func BarWidth(width int) BarOption { return func(s *bState) { - s.width = width + s.reqWidth = width } } @@ -77,19 +77,22 @@ func BarFillerClearOnComplete() BarOption { // BarFillerOnComplete replaces bar's filler with message, on complete event. func BarFillerOnComplete(message string) BarOption { - return func(s *bState) { - s.filler = makeBarFillerOnComplete(s.baseF, message) - } + return BarFillerMiddleware(func(base BarFiller) BarFiller { + return BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) { + if st.Completed { + io.WriteString(w, message) + } else { + base.Fill(w, reqWidth, st) + } + }) + }) } -func makeBarFillerOnComplete(filler BarFiller, message string) BarFiller { - return BarFillerFunc(func(w io.Writer, width int, st *decor.Statistics) { - if st.Completed { - io.WriteString(w, message) - } else { - filler.Fill(w, width, st) - } - }) +// BarFillerMiddleware provides a way to augment default BarFiller. +func BarFillerMiddleware(middle func(BarFiller) BarFiller) BarOption { + return func(s *bState) { + s.middleware = middle + } } // BarPriority sets bar's priority. Zero is highest priority, i.e. bar @@ -103,21 +106,20 @@ func BarPriority(priority int) BarOption { // BarExtender is an option to extend bar to the next new line, with // arbitrary output. -func BarExtender(extender BarFiller) BarOption { - if extender == nil { +func BarExtender(filler BarFiller) BarOption { + if filler == nil { return nil } return func(s *bState) { - s.extender = makeExtFunc(extender) + s.extender = makeExtFunc(filler) } } -func makeExtFunc(extender BarFiller) extFunc { +func makeExtFunc(filler BarFiller) extFunc { buf := new(bytes.Buffer) - nl := []byte("\n") - return func(r io.Reader, tw int, st *decor.Statistics) (io.Reader, int) { - extender.Fill(buf, tw, st) - return io.MultiReader(r, buf), bytes.Count(buf.Bytes(), nl) + return func(r io.Reader, reqWidth int, st decor.Statistics) (io.Reader, int) { + filler.Fill(buf, reqWidth, st) + return io.MultiReader(r, buf), bytes.Count(buf.Bytes(), []byte("\n")) } } @@ -139,7 +141,7 @@ func BarStyle(style string) BarOption { SetStyle(string) } return func(s *bState) { - if t, ok := s.baseF.(styleSetter); ok { + if t, ok := s.filler.(styleSetter); ok { t.SetStyle(style) } } @@ -159,7 +161,7 @@ func BarReverse() BarOption { SetReverse(bool) } return func(s *bState) { - if t, ok := s.baseF.(revSetter); ok { + if t, ok := s.filler.(revSetter); ok { t.SetReverse(true) } } @@ -189,7 +191,7 @@ func MakeFillerTypeSpecificBarOption( cb func(interface{}), ) BarOption { return func(s *bState) { - if t, ok := typeChecker(s.baseF); ok { + if t, ok := typeChecker(s.filler); ok { cb(t) } } diff --git a/vendor/github.com/vbauerster/mpb/v5/options.go b/vendor/github.com/vbauerster/mpb/v5/container_option.go similarity index 92% rename from vendor/github.com/vbauerster/mpb/v5/options.go rename to vendor/github.com/vbauerster/mpb/v5/container_option.go index 04887028..fac59e43 100644 --- a/vendor/github.com/vbauerster/mpb/v5/options.go +++ b/vendor/github.com/vbauerster/mpb/v5/container_option.go @@ -21,14 +21,11 @@ func WithWaitGroup(wg *sync.WaitGroup) ContainerOption { } } -// WithWidth sets container width. Default is 80. Bars inherit this -// width, as long as no BarWidth is applied. -func WithWidth(w int) ContainerOption { +// WithWidth sets container width. If not set underlying bars will +// occupy whole term width. +func WithWidth(width int) ContainerOption { return func(s *pState) { - if w < 0 { - return - } - s.width = w + s.reqWidth = width } } diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go index 9ec1ec66..bb503360 100644 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go +++ b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go @@ -7,7 +7,7 @@ import ( "io" "os" - "golang.org/x/crypto/ssh/terminal" + "github.com/mattn/go-isatty" ) // NotATTY not a TeleTYpewriter error. @@ -30,13 +30,14 @@ func New(out io.Writer) *Writer { w := &Writer{out: out} if f, ok := out.(*os.File); ok { w.fd = f.Fd() - w.isTerminal = terminal.IsTerminal(int(w.fd)) + w.isTerminal = isatty.IsTerminal(w.fd) } return w } // Flush flushes the underlying buffer. func (w *Writer) Flush(lineCount int) (err error) { + // some terminals interpret clear 0 lines as clear 1 if w.lineCount > 0 { w.clearLines() } @@ -63,9 +64,9 @@ func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { // GetWidth returns width of underlying terminal. func (w *Writer) GetWidth() (int, error) { - if w.isTerminal { - tw, _, err := terminal.GetSize(int(w.fd)) - return tw, err + if !w.isTerminal { + return -1, NotATTY } - return -1, NotATTY + tw, _, err := GetSize(w.fd) + return tw, err } diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go index 3fb8b7d7..e836cec3 100644 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go +++ b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go @@ -2,8 +2,21 @@ package cwriter -import "fmt" +import ( + "fmt" + + "golang.org/x/sys/unix" +) func (w *Writer) clearLines() { fmt.Fprintf(w.out, cuuAndEd, w.lineCount) } + +// GetSize returns the dimensions of the given terminal. +func GetSize(fd uintptr) (width, height int, err error) { + ws, err := unix.IoctlGetWinsize(int(fd), unix.TIOCGWINSZ) + if err != nil { + return -1, -1, err + } + return int(ws.Col), int(ws.Row), nil +} diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go index 71252890..7a3ed5bc 100644 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go +++ b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go @@ -14,7 +14,6 @@ var ( procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo") procSetConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition") procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW") - procFillConsoleOutputAttribute = kernel32.NewProc("FillConsoleOutputAttribute") ) type coord struct { @@ -41,8 +40,9 @@ func (w *Writer) clearLines() { if !w.isTerminal { fmt.Fprintf(w.out, cuuAndEd, w.lineCount) } - var info consoleScreenBufferInfo - procGetConsoleScreenBufferInfo.Call(w.fd, uintptr(unsafe.Pointer(&info))) + + info := new(consoleScreenBufferInfo) + procGetConsoleScreenBufferInfo.Call(w.fd, uintptr(unsafe.Pointer(info))) info.cursorPosition.y -= int16(w.lineCount) if info.cursorPosition.y < 0 { @@ -51,10 +51,19 @@ func (w *Writer) clearLines() { procSetConsoleCursorPosition.Call(w.fd, uintptr(uint32(uint16(info.cursorPosition.y))<<16|uint32(uint16(info.cursorPosition.x)))) // clear the lines - cursor := coord{ + cursor := &coord{ x: info.window.left, y: info.cursorPosition.y, } count := uint32(info.size.x) * uint32(w.lineCount) - procFillConsoleOutputCharacter.Call(w.fd, uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(new(uint32)))) + procFillConsoleOutputCharacter.Call(w.fd, uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(cursor)), uintptr(unsafe.Pointer(new(uint32)))) +} + +// GetSize returns the visible dimensions of the given terminal. +// +// These dimensions don't include any scrollback buffer height. +func GetSize(fd uintptr) (width, height int, err error) { + info := new(consoleScreenBufferInfo) + procGetConsoleScreenBufferInfo.Call(fd, uintptr(unsafe.Pointer(info))) + return int(info.window.right - info.window.left), int(info.window.bottom - info.window.top), nil } diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/any.go b/vendor/github.com/vbauerster/mpb/v5/decor/any.go index bf9cf51a..39518f59 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/any.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/any.go @@ -1,21 +1,21 @@ package decor // Any decorator displays text, that can be changed during decorator's -// lifetime via provided func call back. +// lifetime via provided DecorFunc. // -// `f` call back which provides string to display +// `fn` DecorFunc callback // // `wcc` optional WC config // -func Any(f func(*Statistics) string, wcc ...WC) Decorator { - return &any{initWC(wcc...), f} +func Any(fn DecorFunc, wcc ...WC) Decorator { + return &any{initWC(wcc...), fn} } type any struct { WC - f func(*Statistics) string + fn DecorFunc } -func (d *any) Decor(s *Statistics) string { - return d.FormatMsg(d.f(s)) +func (d *any) Decor(s Statistics) string { + return d.FormatMsg(d.fn(s)) } diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/counters.go b/vendor/github.com/vbauerster/mpb/v5/decor/counters.go index 297bf937..010ec371 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/counters.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/counters.go @@ -46,21 +46,21 @@ func Counters(unit int, pairFmt string, wcc ...WC) Decorator { return Any(chooseSizeProducer(unit, pairFmt), wcc...) } -func chooseSizeProducer(unit int, format string) func(*Statistics) string { +func chooseSizeProducer(unit int, format string) DecorFunc { if format == "" { format = "%d / %d" } switch unit { case UnitKiB: - return func(s *Statistics) string { + return func(s Statistics) string { return fmt.Sprintf(format, SizeB1024(s.Current), SizeB1024(s.Total)) } case UnitKB: - return func(s *Statistics) string { + return func(s Statistics) string { return fmt.Sprintf(format, SizeB1000(s.Current), SizeB1000(s.Total)) } default: - return func(s *Statistics) string { + return func(s Statistics) string { return fmt.Sprintf(format, s.Current, s.Total) } } diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go b/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go index 5bca63d5..e81fae36 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go @@ -3,9 +3,9 @@ package decor import ( "fmt" "time" - "unicode/utf8" "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" ) const ( @@ -47,22 +47,32 @@ const ( // Statistics consists of progress related statistics, that Decorator // may need. type Statistics struct { - ID int - Completed bool - Total int64 - Current int64 + ID int + AvailableWidth int + Total int64 + Current int64 + Refill int64 + Completed bool } // Decorator interface. -// Implementors should embed WC type, that way only single method -// Decor(*Statistics) needs to be implemented, the rest will be handled -// by WC type. +// Most of the time there is no need to implement this interface +// manually, as decor package already provides a wide range of decorators +// which implement this interface. If however built-in decorators don't +// meet your needs, you're free to implement your own one by implementing +// this particular interface. The easy way to go is to convert a +// `DecorFunc` into a `Decorator` interface by using provided +// `func Any(DecorFunc, ...WC) Decorator`. type Decorator interface { Configurator Synchronizer - Decor(*Statistics) string + Decor(Statistics) string } +// DecorFunc func type. +// To be used with `func Any`(DecorFunc, ...WC) Decorator`. +type DecorFunc func(Statistics) string + // Synchronizer interface. // All decorators implement this interface implicitly. Its Sync // method exposes width sync channel, if DSyncWidth bit is set. @@ -117,38 +127,35 @@ var ( // W represents width and C represents bit set of width related config. // A decorator should embed WC, to enable width synchronization. type WC struct { - W int - C int - dynFormat string - wsync chan int + W int + C int + fill func(s string, w int) string + wsync chan int } // FormatMsg formats final message according to WC.W and WC.C. // Should be called by any Decorator implementation. func (wc *WC) FormatMsg(msg string) string { - var format string - runeCount := utf8.RuneCountInString(stripansi.Strip(msg)) - ansiCount := utf8.RuneCountInString(msg) - runeCount + pureWidth := runewidth.StringWidth(msg) + stripWidth := runewidth.StringWidth(stripansi.Strip(msg)) + maxCell := wc.W if (wc.C & DSyncWidth) != 0 { + cellCount := stripWidth if (wc.C & DextraSpace) != 0 { - runeCount++ + cellCount++ } - wc.wsync <- runeCount - max := <-wc.wsync - format = fmt.Sprintf(wc.dynFormat, ansiCount+max) - } else { - format = fmt.Sprintf(wc.dynFormat, ansiCount+wc.W) + wc.wsync <- cellCount + maxCell = <-wc.wsync } - return fmt.Sprintf(format, msg) + return wc.fill(msg, maxCell+(pureWidth-stripWidth)) } // Init initializes width related config. func (wc *WC) Init() WC { - wc.dynFormat = "%%" + wc.fill = runewidth.FillLeft if (wc.C & DidentRight) != 0 { - wc.dynFormat += "-" + wc.fill = runewidth.FillRight } - wc.dynFormat += "%ds" if (wc.C & DSyncWidth) != 0 { // it's deliberate choice to override wsync on each Init() call, // this way globals like WCSyncSpace can be reused diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go b/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go index c9999a3b..e389f158 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go @@ -25,11 +25,11 @@ func Elapsed(style TimeStyle, wcc ...WC) Decorator { func NewElapsed(style TimeStyle, startTime time.Time, wcc ...WC) Decorator { var msg string producer := chooseTimeProducer(style) - f := func(s *Statistics) string { + fn := func(s Statistics) string { if !s.Completed { msg = producer(time.Since(startTime)) } return msg } - return Any(f, wcc...) + return Any(fn, wcc...) } diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/eta.go b/vendor/github.com/vbauerster/mpb/v5/decor/eta.go index 6cb27a24..d03caa73 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/eta.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/eta.go @@ -63,7 +63,7 @@ type movingAverageETA struct { producer func(time.Duration) string } -func (d *movingAverageETA) Decor(s *Statistics) string { +func (d *movingAverageETA) Decor(s Statistics) string { v := math.Round(d.average.Value()) remaining := time.Duration((s.Total - s.Current) * int64(v)) if d.normalizer != nil { @@ -117,7 +117,7 @@ type averageETA struct { producer func(time.Duration) string } -func (d *averageETA) Decor(s *Statistics) string { +func (d *averageETA) Decor(s Statistics) string { var remaining time.Duration if s.Current != 0 { durPerItem := float64(time.Since(d.startTime)) / float64(s.Current) diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/merge.go b/vendor/github.com/vbauerster/mpb/v5/decor/merge.go index 520f13a7..e41406a6 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/merge.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/merge.go @@ -1,9 +1,10 @@ package decor import ( - "fmt" "strings" - "unicode/utf8" + + "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" ) // Merge wraps its decorator argument with intention to sync width @@ -64,18 +65,18 @@ func (d *mergeDecorator) Base() Decorator { return d.Decorator } -func (d *mergeDecorator) Decor(s *Statistics) string { +func (d *mergeDecorator) Decor(s Statistics) string { msg := d.Decorator.Decor(s) - msgLen := utf8.RuneCountInString(msg) + pureWidth := runewidth.StringWidth(msg) + stripWidth := runewidth.StringWidth(stripansi.Strip(msg)) + cellCount := stripWidth if (d.wc.C & DextraSpace) != 0 { - msgLen++ + cellCount++ } - var total int - max := utf8.RuneCountInString(d.placeHolders[0].FormatMsg("")) - total += max - pw := (msgLen - max) / len(d.placeHolders) - rem := (msgLen - max) % len(d.placeHolders) + total := runewidth.StringWidth(d.placeHolders[0].FormatMsg("")) + pw := (cellCount - total) / len(d.placeHolders) + rem := (cellCount - total) % len(d.placeHolders) var diff int for i := 1; i < len(d.placeHolders); i++ { @@ -87,20 +88,20 @@ func (d *mergeDecorator) Decor(s *Statistics) string { width = 0 } } - max = utf8.RuneCountInString(ph.FormatMsg(strings.Repeat(" ", width))) + max := runewidth.StringWidth(ph.FormatMsg(strings.Repeat(" ", width))) total += max diff = max - pw } d.wc.wsync <- pw + rem - max = <-d.wc.wsync - return fmt.Sprintf(fmt.Sprintf(d.wc.dynFormat, max+total), msg) + max := <-d.wc.wsync + return d.wc.fill(msg, max+total+(pureWidth-stripWidth)) } type placeHolderDecorator struct { WC } -func (d *placeHolderDecorator) Decor(*Statistics) string { +func (d *placeHolderDecorator) Decor(Statistics) string { return "" } diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/name.go b/vendor/github.com/vbauerster/mpb/v5/decor/name.go index a7d477e0..3af31125 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/name.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/name.go @@ -8,5 +8,5 @@ package decor // `wcc` optional WC config // func Name(str string, wcc ...WC) Decorator { - return Any(func(*Statistics) string { return str }, wcc...) + return Any(func(Statistics) string { return str }, wcc...) } diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go b/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go index 0a1526bf..f46b19ab 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go @@ -24,7 +24,7 @@ type onCompleteWrapper struct { msg string } -func (d *onCompleteWrapper) Decor(s *Statistics) string { +func (d *onCompleteWrapper) Decor(s Statistics) string { if s.Completed { wc := d.GetConf() return wc.FormatMsg(d.msg) diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go b/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go index 65ca7d31..d6314a61 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go @@ -50,7 +50,7 @@ func NewPercentage(format string, wcc ...WC) Decorator { if format == "" { format = "% d" } - f := func(s *Statistics) string { + f := func(s Statistics) string { p := internal.Percentage(s.Total, s.Current, 100) return fmt.Sprintf(format, percentageType(p)) } diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/speed.go b/vendor/github.com/vbauerster/mpb/v5/decor/speed.go index 8a48e3f5..634edabf 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/speed.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/speed.go @@ -78,7 +78,7 @@ type movingAverageSpeed struct { msg string } -func (d *movingAverageSpeed) Decor(s *Statistics) string { +func (d *movingAverageSpeed) Decor(s Statistics) string { if !s.Completed { var speed float64 if v := d.average.Value(); v > 0 { @@ -140,7 +140,7 @@ type averageSpeed struct { msg string } -func (d *averageSpeed) Decor(s *Statistics) string { +func (d *averageSpeed) Decor(s Statistics) string { if !s.Completed { speed := float64(s.Current) / float64(time.Since(d.startTime)) d.msg = d.producer(speed * 1e9) diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go b/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go index abfb2f76..6871639d 100644 --- a/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go +++ b/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go @@ -12,7 +12,7 @@ func Spinner(frames []string, wcc ...WC) Decorator { frames = defaultSpinnerStyle } var count uint - f := func(s *Statistics) string { + f := func(s Statistics) string { frame := frames[count%uint(len(frames))] count++ return frame diff --git a/vendor/github.com/vbauerster/mpb/v5/go.mod b/vendor/github.com/vbauerster/mpb/v5/go.mod index 1d8d5293..389a19d5 100644 --- a/vendor/github.com/vbauerster/mpb/v5/go.mod +++ b/vendor/github.com/vbauerster/mpb/v5/go.mod @@ -3,8 +3,9 @@ module github.com/vbauerster/mpb/v5 require ( github.com/VividCortex/ewma v1.1.1 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d - golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 - golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect + github.com/mattn/go-isatty v0.0.12 + github.com/mattn/go-runewidth v0.0.9 + golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 ) go 1.14 diff --git a/vendor/github.com/vbauerster/mpb/v5/go.sum b/vendor/github.com/vbauerster/mpb/v5/go.sum index 99ca1bf6..dcaa8c55 100644 --- a/vendor/github.com/vbauerster/mpb/v5/go.sum +++ b/vendor/github.com/vbauerster/mpb/v5/go.sum @@ -2,12 +2,10 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU= -golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/vbauerster/mpb/v5/internal/width.go b/vendor/github.com/vbauerster/mpb/v5/internal/width.go new file mode 100644 index 00000000..35d52898 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v5/internal/width.go @@ -0,0 +1,8 @@ +package internal + +func WidthForBarFiller(reqWidth, available int) int { + if reqWidth <= 0 || reqWidth >= available { + return available + } + return reqWidth +} diff --git a/vendor/github.com/vbauerster/mpb/v5/progress.go b/vendor/github.com/vbauerster/mpb/v5/progress.go index a366b929..ac1ce50a 100644 --- a/vendor/github.com/vbauerster/mpb/v5/progress.go +++ b/vendor/github.com/vbauerster/mpb/v5/progress.go @@ -19,8 +19,6 @@ import ( const ( // default RefreshRate prr = 120 * time.Millisecond - // default width - pwidth = 80 ) // Progress represents the container that renders Progress bars @@ -46,7 +44,7 @@ type pState struct { // following are provided/overrided by user idCount int - width int + reqWidth int popCompleted bool rr time.Duration uwg *sync.WaitGroup @@ -70,7 +68,6 @@ func New(options ...ContainerOption) *Progress { func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress { s := &pState{ bHeap: priorityQueue{}, - width: pwidth, rr: prr, parkedBars: make(map[*Bar]*Bar), output: os.Stdout, @@ -113,7 +110,7 @@ func (p *Progress) AddSpinner(total int64, alignment SpinnerAlignment, options . // Panics if *Progress instance is done, i.e. called after *Progress.Wait(). func (p *Progress) Add(total int64, filler BarFiller, options ...BarOption) *Bar { if filler == nil { - filler = NewBarFiller(DefaultBarStyle, false) + filler = BarFillerFunc(func(io.Writer, int, decor.Statistics) {}) } p.bwg.Add(1) result := make(chan *Bar) @@ -215,14 +212,46 @@ func (p *Progress) serve(s *pState, cw *cwriter.Writer) { op(s) case <-p.refreshCh: if err := s.render(cw); err != nil { - go p.dlogger.Println(err) + p.dlogger.Println(err) } case <-s.shutdownNotifier: + if s.heapUpdated { + if err := s.render(cw); err != nil { + p.dlogger.Println(err) + } + } return } } } +func (s *pState) newTicker(done <-chan struct{}) chan time.Time { + ch := make(chan time.Time) + if s.shutdownNotifier == nil { + s.shutdownNotifier = make(chan struct{}) + } + go func() { + if s.renderDelay != nil { + <-s.renderDelay + } + if s.refreshSrc == nil { + ticker := time.NewTicker(s.rr) + defer ticker.Stop() + s.refreshSrc = ticker.C + } + for { + select { + case tick := <-s.refreshSrc: + ch <- tick + case <-done: + close(s.shutdownNotifier) + return + } + } + }() + return ch +} + func (s *pState) render(cw *cwriter.Writer) error { if s.heapUpdated { s.updateSyncMatrix() @@ -233,7 +262,7 @@ func (s *pState) render(cw *cwriter.Writer) error { tw, err := cw.GetWidth() if err != nil { - tw = s.width + tw = s.reqWidth } for i := 0; i < s.bHeap.Len(); i++ { bar := s.bHeap[i] @@ -250,11 +279,16 @@ func (s *pState) flush(cw *cwriter.Writer) error { b := heap.Pop(&s.bHeap).(*Bar) cw.ReadFrom(<-b.frameCh) if b.toShutdown { - // shutdown at next flush - // this ensures no bar ends up with less than 100% rendered - defer func() { + if b.recoveredPanic != nil { s.barShutdownQueue = append(s.barShutdownQueue, b) - }() + b.toShutdown = false + } else { + // shutdown at next flush + // this ensures no bar ends up with less than 100% rendered + defer func() { + s.barShutdownQueue = append(s.barShutdownQueue, b) + }() + } } lineCount += b.extendedLines + 1 bm[b] = struct{}{} @@ -295,33 +329,6 @@ func (s *pState) flush(cw *cwriter.Writer) error { return cw.Flush(lineCount) } -func (s *pState) newTicker(done <-chan struct{}) chan time.Time { - ch := make(chan time.Time) - if s.shutdownNotifier == nil { - s.shutdownNotifier = make(chan struct{}) - } - go func() { - if s.renderDelay != nil { - <-s.renderDelay - } - if s.refreshSrc == nil { - ticker := time.NewTicker(s.rr) - defer ticker.Stop() - s.refreshSrc = ticker.C - } - for { - select { - case tick := <-s.refreshSrc: - ch <- tick - case <-done: - close(s.shutdownNotifier) - return - } - } - }() - return ch -} - func (s *pState) updateSyncMatrix() { s.pMatrix = make(map[int][]chan int) s.aMatrix = make(map[int][]chan int) @@ -342,16 +349,13 @@ func (s *pState) updateSyncMatrix() { func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState { bs := &bState{ - total: total, - baseF: extractBaseFiller(filler), - filler: filler, - priority: s.idCount, id: s.idCount, - width: s.width, + priority: s.idCount, + reqWidth: s.reqWidth, + total: total, + filler: filler, + extender: func(r io.Reader, _ int, _ decor.Statistics) (io.Reader, int) { return r, 0 }, debugOut: s.debugOut, - extender: func(r io.Reader, _ int, _ *decor.Statistics) (io.Reader, int) { - return r, 0 - }, } for _, opt := range options { @@ -360,13 +364,18 @@ func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOptio } } + if bs.middleware != nil { + bs.filler = bs.middleware(filler) + bs.middleware = nil + } + if s.popCompleted && !bs.noPop { bs.priority = -1 } - bs.bufP = bytes.NewBuffer(make([]byte, 0, bs.width)) - bs.bufB = bytes.NewBuffer(make([]byte, 0, bs.width)) - bs.bufA = bytes.NewBuffer(make([]byte, 0, bs.width)) + bs.bufP = bytes.NewBuffer(make([]byte, 0, 128)) + bs.bufB = bytes.NewBuffer(make([]byte, 0, 256)) + bs.bufA = bytes.NewBuffer(make([]byte, 0, 128)) return bs } @@ -387,13 +396,3 @@ func syncWidth(matrix map[int][]chan int) { }() } } - -func extractBaseFiller(f BarFiller) BarFiller { - type wrapper interface { - Base() BarFiller - } - if f, ok := f.(wrapper); ok { - return extractBaseFiller(f.Base()) - } - return f -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7c053fad..31d8f92f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -39,7 +39,7 @@ github.com/containerd/containerd/log github.com/containerd/containerd/platforms # github.com/containers/common v0.14.0 github.com/containers/common/pkg/auth -# github.com/containers/image/v5 v5.4.4 +# github.com/containers/image/v5 v5.5.1 github.com/containers/image/v5/copy github.com/containers/image/v5/directory github.com/containers/image/v5/directory/explicitfilepath @@ -207,7 +207,7 @@ github.com/hashicorp/golang-lru/simplelru github.com/imdario/mergo # github.com/inconshreveable/mousetrap v1.0.0 github.com/inconshreveable/mousetrap -# github.com/klauspost/compress v1.10.7 +# github.com/klauspost/compress v1.10.8 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -222,6 +222,10 @@ github.com/konsorten/go-windows-terminal-sequences github.com/kr/pretty # github.com/kr/text v0.1.0 github.com/kr/text +# github.com/mattn/go-isatty v0.0.12 +github.com/mattn/go-isatty +# github.com/mattn/go-runewidth v0.0.9 +github.com/mattn/go-runewidth # github.com/mattn/go-shellwords v1.0.10 github.com/mattn/go-shellwords # github.com/matttproud/golang_protobuf_extensions v1.0.1 @@ -295,7 +299,7 @@ github.com/ulikunitz/xz/lzma github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm github.com/vbatts/tar-split/tar/storage -# github.com/vbauerster/mpb/v5 v5.0.4 +# github.com/vbauerster/mpb/v5 v5.2.2 github.com/vbauerster/mpb/v5 github.com/vbauerster/mpb/v5/cwriter github.com/vbauerster/mpb/v5/decor