Merge pull request #298 from mtrmac/openpgp

Vendor in non-gpgme support, and update tests and build machinery

Makefile

@@ -48,12 +48,12 @@ all: binary docs
 binary: cmd/skopeo
 	docker build ${DOCKER_BUILD_ARGS} -f Dockerfile.build -t skopeobuildimage .
 	docker run --rm --security-opt label:disable -v $$(pwd):/src/github.com/projectatomic/skopeo \
-		skopeobuildimage make binary-local $(if $(DEBUG),DEBUG=$(DEBUG))
+		skopeobuildimage make binary-local $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)'
 
 binary-static: cmd/skopeo
 	docker build ${DOCKER_BUILD_ARGS} -f Dockerfile.build -t skopeobuildimage .
 	docker run --rm --security-opt label:disable -v $$(pwd):/src/github.com/projectatomic/skopeo \
-		skopeobuildimage make binary-local-static $(if $(DEBUG),DEBUG=$(DEBUG))
+		skopeobuildimage make binary-local-static $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)'
 
 # Build w/o using Docker containers
 binary-local:
@@ -95,11 +95,11 @@ check: validate test-unit test-integration
 
 # The tests can run out of entropy and block in containers, so replace /dev/random.
 test-integration: build-container
-	$(DOCKER_RUN_DOCKER) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 hack/make.sh test-integration'
+	$(DOCKER_RUN_DOCKER) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-integration'
 
 test-unit: build-container
 	# Just call (make test unit-local) here instead of worrying about environment differences, e.g. GO15VENDOREXPERIMENT.
-	$(DOCKER_RUN_DOCKER) make test-unit-local
+	$(DOCKER_RUN_DOCKER) make test-unit-local BUILDTAGS='$(BUILDTAGS)'
 
 validate: build-container
 	$(DOCKER_RUN_DOCKER) hack/make.sh validate-git-marks validate-gofmt validate-lint validate-vet

cmd/skopeo/signing.go

@@ -28,6 +28,7 @@ func standaloneSign(context *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("Error initializing GPG: %v", err)
 	}
+	defer mech.Close()
 	signature, err := signature.SignDockerManifest(manifest, dockerReference, mech, fingerprint)
 	if err != nil {
 		return fmt.Errorf("Error creating signature: %v", err)
@@ -74,6 +75,7 @@ func standaloneVerify(context *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("Error initializing GPG: %v", err)
 	}
+	defer mech.Close()
 	sig, err := signature.VerifyDockerManifestSignature(unverifiedSignature, unverifiedManifest, expectedDockerReference, mech, expectedFingerprint)
 	if err != nil {
 		return fmt.Errorf("Error verifying signature: %v", err)

cmd/skopeo/signing_test.go

@@ -31,6 +31,13 @@ func assertTestFailed(t *testing.T, stdout string, err error, substring string)
 }
 
 func TestStandaloneSign(t *testing.T) {
+	mech, _, err := signature.NewEphemeralGPGSigningMechanism([]byte{})
+	require.NoError(t, err)
+	defer mech.Close()
+	if err := mech.SupportsSigning(); err != nil {
+		t.Skipf("Signing not supported: %v", err)
+	}
+
 	manifestPath := "fixtures/image.manifest.json"
 	dockerReference := "testing/manifest"
 	os.Setenv("GNUPGHOME", "fixtures")
@@ -76,17 +83,18 @@ func TestStandaloneSign(t *testing.T) {
 	defer os.Remove(sigOutput.Name())
 	out, err = runSkopeo("standalone-sign", "-o", sigOutput.Name(),
 		manifestPath, dockerReference, fixturesTestKeyFingerprint)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	assert.Empty(t, out)
 
 	sig, err := ioutil.ReadFile(sigOutput.Name())
 	require.NoError(t, err)
 	manifest, err := ioutil.ReadFile(manifestPath)
 	require.NoError(t, err)
-	mech, err := signature.NewGPGSigningMechanism()
+	mech, err = signature.NewGPGSigningMechanism()
 	require.NoError(t, err)
+	defer mech.Close()
 	verified, err := signature.VerifyDockerManifestSignature(sig, manifest, dockerReference, mech, fixturesTestKeyFingerprint)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, dockerReference, verified.DockerReference)
 	assert.Equal(t, fixturesTestImageManifestDigest, verified.DockerManifestDigest)
 }

hack/make.sh

@@ -72,10 +72,10 @@ TESTFLAGS+=" -test.timeout=10m"
 go_test_dir() {
 	dir=$1
 	(
-		echo '+ go test' $TESTFLAGS "${SKOPEO_PKG}${dir#.}"
+		echo '+ go test' $TESTFLAGS ${BUILDTAGS:+-tags "$BUILDTAGS"} "${SKOPEO_PKG}${dir#.}"
 		cd "$dir"
 		export DEST="$ABS_DEST" # we're in a subshell, so this is safe -- our integration-cli tests need DEST, and "cd" screws it up
-		go test $TESTFLAGS
+		go test $TESTFLAGS ${BUILDTAGS:+-tags "$BUILDTAGS"}
 	)
 }
 

hack/make/test-integration

@@ -8,7 +8,7 @@ bundle_test_integration() {
 
 # subshell so that we can export PATH without breaking other things
 (
-	make binary-local
+	make binary-local ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"}
 	make install
 	export GO15VENDOREXPERIMENT=1
 	bundle_test_integration

integration/copy_test.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/containers/image/manifest"
+	"github.com/containers/image/signature"
 	"github.com/go-check/check"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-tools/image"
@@ -242,6 +243,13 @@ func (s *CopySuite) TestCopyOCIRoundTrip(c *check.C) {
 
 // --sign-by and --policy copy, primarily using atomic:
 func (s *CopySuite) TestCopySignatures(c *check.C) {
+	mech, _, err := signature.NewEphemeralGPGSigningMechanism([]byte{})
+	c.Assert(err, check.IsNil)
+	defer mech.Close()
+	if err := mech.SupportsSigning(); err != nil { // FIXME? Test that verification and policy enforcement works, using signatures from fixtures
+		c.Skip(fmt.Sprintf("Signing not supported: %v", err))
+	}
+
 	dir, err := ioutil.TempDir("", "signatures-dest")
 	c.Assert(err, check.IsNil)
 	defer os.RemoveAll(dir)
@@ -291,6 +299,13 @@ func (s *CopySuite) TestCopySignatures(c *check.C) {
 
 // --policy copy for dir: sources
 func (s *CopySuite) TestCopyDirSignatures(c *check.C) {
+	mech, _, err := signature.NewEphemeralGPGSigningMechanism([]byte{})
+	c.Assert(err, check.IsNil)
+	defer mech.Close()
+	if err := mech.SupportsSigning(); err != nil { // FIXME? Test that verification and policy enforcement works, using signatures from fixtures
+		c.Skip(fmt.Sprintf("Signing not supported: %v", err))
+	}
+
 	topDir, err := ioutil.TempDir("", "dir-signatures-top")
 	c.Assert(err, check.IsNil)
 	defer os.RemoveAll(topDir)
@@ -390,6 +405,13 @@ func findRegularFiles(c *check.C, root string) []string {
 
 // --sign-by and policy use for docker: with sigstore
 func (s *CopySuite) TestCopyDockerSigstore(c *check.C) {
+	mech, _, err := signature.NewEphemeralGPGSigningMechanism([]byte{})
+	c.Assert(err, check.IsNil)
+	defer mech.Close()
+	if err := mech.SupportsSigning(); err != nil { // FIXME? Test that verification and policy enforcement works, using signatures from fixtures
+		c.Skip(fmt.Sprintf("Signing not supported: %v", err))
+	}
+
 	const ourRegistry = "docker://" + v2DockerRegistryURL + "/"
 
 	tmpDir, err := ioutil.TempDir("", "signatures-sigstore")

integration/signing_test.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/containers/image/signature"
 	"github.com/go-check/check"
 )
 
@@ -36,7 +37,14 @@ func findFingerprint(lineBytes []byte) (string, error) {
 }
 
 func (s *SigningSuite) SetUpTest(c *check.C) {
-	_, err := exec.LookPath(skopeoBinary)
+	mech, _, err := signature.NewEphemeralGPGSigningMechanism([]byte{})
+	c.Assert(err, check.IsNil)
+	defer mech.Close()
+	if err := mech.SupportsSigning(); err != nil { // FIXME? Test that verification and policy enforcement works, using signatures from fixtures
+		c.Skip(fmt.Sprintf("Signing not supported: %v", err))
+	}
+
+	_, err = exec.LookPath(skopeoBinary)
 	c.Assert(err, check.IsNil)
 
 	s.gpgHome, err = ioutil.TempDir("", "skopeo-gpg")

vendor.conf

@@ -19,13 +19,14 @@ github.com/gorilla/mux e444e69cbd
 github.com/docker/go-units 8a7beacffa3009a9ac66bad506b18ffdd110cf97
 golang.org/x/net master
 # end docker deps
+golang.org/x/text master
 github.com/docker/distribution master
 github.com/docker/libtrust master
 github.com/opencontainers/runc master
 github.com/opencontainers/image-spec v1.0.0-rc4
 # -- start OCI image validation requirements.
 github.com/opencontainers/runtime-spec v1.0.0-rc4
-github.com/opencontainers/image-tools master
+github.com/opencontainers/image-tools v0.1.0
 github.com/xeipuuv/gojsonschema master
 github.com/xeipuuv/gojsonreference master
 github.com/xeipuuv/gojsonpointer master
@@ -41,3 +42,4 @@ github.com/imdario/mergo 6633656539c1639d9d78127b7d47c622b5d7b6dc
 # containers/storage's dependencies that aren't already being pulled in
 github.com/mistifyio/go-zfs 22c9b32c84eb0d0c6f4043b6e90fc94073de92fa
 github.com/pborman/uuid v1.0
+github.com/opencontainers/selinux/go-selinux/label master

vendor/github.com/containers/image/README.md

@@ -32,6 +32,16 @@ libraries you should use with this package in your own projects.
 What this project tests against dependencies-wise is located
 [here](https://github.com/containers/image/blob/master/vendor.conf).
 
+## Building
+
+For ordinary use, `go build ./...` is sufficient.
+
+When developing this library, please use `make` to take advantage of the tests and validation.
+
+Optionally, you can use the `containers_image_openpgp` build tag (using `go build -tags …`, or `make … BUILDTAGS=…`).
+This will use a Golang-only OpenPGP implementation for signature verification instead of the default cgo/gpgme-based implementation;
+the primary downside is that creating new signatures with the Golang-only implementation is not supported.
+
 ## License
 
 ASL 2.0

vendor/github.com/containers/image/copy/copy.go

@@ -235,6 +235,11 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
 		if err != nil {
 			return errors.Wrap(err, "Error initializing GPG")
 		}
+		defer mech.Close()
+		if err := mech.SupportsSigning(); err != nil {
+			return errors.Wrap(err, "Signing not supported")
+		}
+
 		dockerReference := dest.Reference().DockerReference()
 		if dockerReference == nil {
 			return errors.Errorf("Cannot determine canonical Docker reference for destination %s", transports.ImageName(dest.Reference()))

vendor/github.com/containers/image/docker/docker_image_dest.go

@@ -42,7 +42,7 @@ type dockerImageDestination struct {
 
 // newImageDestination creates a new ImageDestination for the specified image reference.
 func newImageDestination(ctx *types.SystemContext, ref dockerReference) (types.ImageDestination, error) {
-	c, err := newDockerClient(ctx, ref, true, "push")
+	c, err := newDockerClient(ctx, ref, true, "pull,push")
 	if err != nil {
 		return nil, err
 	}
@@ -132,7 +132,7 @@ func (d *dockerImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobI
 	tee := io.TeeReader(stream, io.MultiWriter(digester.Hash(), sizeCounter))
 	res, err = d.c.makeRequestToResolvedURL("PATCH", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, tee, inputInfo.Size, true)
 	if err != nil {
-		logrus.Debugf("Error uploading layer chunked, response %#v", *res)
+		logrus.Debugf("Error uploading layer chunked, response %#v", res)
 		return types.BlobInfo{}, err
 	}
 	defer res.Body.Close()

vendor/github.com/containers/image/signature/json.go

@@ -14,64 +14,6 @@ func (err jsonFormatError) Error() string {
 	return string(err)
 }
 
-// validateExactMapKeys returns an error if the keys of m are not exactly expectedKeys, which must be pairwise distinct
-func validateExactMapKeys(m map[string]interface{}, expectedKeys ...string) error {
-	if len(m) != len(expectedKeys) {
-		return jsonFormatError("Unexpected keys in a JSON object")
-	}
-
-	for _, k := range expectedKeys {
-		if _, ok := m[k]; !ok {
-			return jsonFormatError(fmt.Sprintf("Key %s missing in a JSON object", k))
-		}
-	}
-	// Assuming expectedKeys are pairwise distinct, we know m contains len(expectedKeys) different values in expectedKeys.
-	return nil
-}
-
-// int64Field returns a member fieldName of m, if it is an int64, or an error.
-func int64Field(m map[string]interface{}, fieldName string) (int64, error) {
-	untyped, ok := m[fieldName]
-	if !ok {
-		return -1, jsonFormatError(fmt.Sprintf("Field %s missing", fieldName))
-	}
-	f, ok := untyped.(float64)
-	if !ok {
-		return -1, jsonFormatError(fmt.Sprintf("Field %s is not a number", fieldName))
-	}
-	v := int64(f)
-	if float64(v) != f {
-		return -1, jsonFormatError(fmt.Sprintf("Field %s is not an integer", fieldName))
-	}
-	return v, nil
-}
-
-// mapField returns a member fieldName of m, if it is a JSON map, or an error.
-func mapField(m map[string]interface{}, fieldName string) (map[string]interface{}, error) {
-	untyped, ok := m[fieldName]
-	if !ok {
-		return nil, jsonFormatError(fmt.Sprintf("Field %s missing", fieldName))
-	}
-	v, ok := untyped.(map[string]interface{})
-	if !ok {
-		return nil, jsonFormatError(fmt.Sprintf("Field %s is not a JSON object", fieldName))
-	}
-	return v, nil
-}
-
-// stringField returns a member fieldName of m, if it is a string, or an error.
-func stringField(m map[string]interface{}, fieldName string) (string, error) {
-	untyped, ok := m[fieldName]
-	if !ok {
-		return "", jsonFormatError(fmt.Sprintf("Field %s missing", fieldName))
-	}
-	v, ok := untyped.(string)
-	if !ok {
-		return "", jsonFormatError(fmt.Sprintf("Field %s is not a string", fieldName))
-	}
-	return v, nil
-}
-
 // paranoidUnmarshalJSONObject unmarshals data as a JSON object, but failing on the slightest unexpected aspect
 // (including duplicated keys, unrecognized keys, and non-matching types). Uses fieldResolver to
 // determine the destination for a field value, which should return a pointer to the destination if valid, or nil if the key is rejected.
@@ -122,3 +64,25 @@ func paranoidUnmarshalJSONObject(data []byte, fieldResolver func(string) interfa
 	}
 	return nil
 }
+
+// paranoidUnmarshalJSONObject unmarshals data as a JSON object, but failing on the slightest unexpected aspect
+// (including duplicated keys, unrecognized keys, and non-matching types). Each of the fields in exactFields
+// must be present exactly once, and none other fields are accepted.
+func paranoidUnmarshalJSONObjectExactFields(data []byte, exactFields map[string]interface{}) error {
+	seenKeys := map[string]struct{}{}
+	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
+		if valuePtr, ok := exactFields[key]; ok {
+			seenKeys[key] = struct{}{}
+			return valuePtr
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	for key := range exactFields {
+		if _, ok := seenKeys[key]; !ok {
+			return jsonFormatError(fmt.Sprintf(`Key "%s" missing in a JSON object`, key))
+		}
+	}
+	return nil
+}

vendor/github.com/containers/image/signature/mechanism.go

@@ -9,19 +9,20 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"github.com/mtrmac/gpgme"
 	"golang.org/x/crypto/openpgp"
 )
 
 // SigningMechanism abstracts a way to sign binary blobs and verify their signatures.
+// Each mechanism should eventually be closed by calling Close().
 // FIXME: Eventually expand on keyIdentity (namespace them between mechanisms to
 // eliminate ambiguities, support CA signatures and perhaps other key properties)
 type SigningMechanism interface {
-	// ImportKeysFromBytes imports public keys from the supplied blob and returns their identities.
-	// The blob is assumed to have an appropriate format (the caller is expected to know which one).
-	// NOTE: This may modify long-term state (e.g. key storage in a directory underlying the mechanism).
-	ImportKeysFromBytes(blob []byte) ([]string, error)
-	// Sign creates a (non-detached) signature of input using keyidentity
+	// Close removes resources associated with the mechanism, if any.
+	Close() error
+	// SupportsSigning returns nil if the mechanism supports signing, or a SigningNotSupportedError.
+	SupportsSigning() error
+	// Sign creates a (non-detached) signature of input using keyIdentity.
+	// Fails with a SigningNotSupportedError if the mechanism does not support signing.
 	Sign(input []byte, keyIdentity string) ([]byte, error)
 	// Verify parses unverifiedSignature and returns the content and the signer's identity
 	Verify(unverifiedSignature []byte) (contents []byte, keyIdentity string, err error)
@@ -33,109 +34,34 @@ type SigningMechanism interface {
 	UntrustedSignatureContents(untrustedSignature []byte) (untrustedContents []byte, shortKeyIdentifier string, err error)
 }
 
-// A GPG/OpenPGP signing mechanism.
-type gpgSigningMechanism struct {
-	ctx *gpgme.Context
+// SigningNotSupportedError is returned when trying to sign using a mechanism which does not support that.
+type SigningNotSupportedError string
+
+func (err SigningNotSupportedError) Error() string {
+	return string(err)
 }
 
-// NewGPGSigningMechanism returns a new GPG/OpenPGP signing mechanism.
+// NewGPGSigningMechanism returns a new GPG/OpenPGP signing mechanism for the user’s default
+// GPG configuration ($GNUPGHOME / ~/.gnupg)
+// The caller must call .Close() on the returned SigningMechanism.
 func NewGPGSigningMechanism() (SigningMechanism, error) {
 	return newGPGSigningMechanismInDirectory("")
 }
 
-// newGPGSigningMechanismInDirectory returns a new GPG/OpenPGP signing mechanism, using optionalDir if not empty.
-func newGPGSigningMechanismInDirectory(optionalDir string) (SigningMechanism, error) {
-	ctx, err := gpgme.New()
-	if err != nil {
-		return nil, err
-	}
-	if err = ctx.SetProtocol(gpgme.ProtocolOpenPGP); err != nil {
-		return nil, err
-	}
-	if optionalDir != "" {
-		err := ctx.SetEngineInfo(gpgme.ProtocolOpenPGP, "", optionalDir)
-		if err != nil {
-			return nil, err
-		}
-	}
-	ctx.SetArmor(false)
-	ctx.SetTextMode(false)
-	return gpgSigningMechanism{ctx: ctx}, nil
+// NewEphemeralGPGSigningMechanism returns a new GPG/OpenPGP signing mechanism which
+// recognizes _only_ public keys from the supplied blob, and returns the identities
+// of these keys.
+// The caller must call .Close() on the returned SigningMechanism.
+func NewEphemeralGPGSigningMechanism(blob []byte) (SigningMechanism, []string, error) {
+	return newEphemeralGPGSigningMechanism(blob)
 }
 
-// ImportKeysFromBytes implements SigningMechanism.ImportKeysFromBytes
-func (m gpgSigningMechanism) ImportKeysFromBytes(blob []byte) ([]string, error) {
-	inputData, err := gpgme.NewDataBytes(blob)
-	if err != nil {
-		return nil, err
-	}
-	res, err := m.ctx.Import(inputData)
-	if err != nil {
-		return nil, err
-	}
-	keyIdentities := []string{}
-	for _, i := range res.Imports {
-		if i.Result == nil {
-			keyIdentities = append(keyIdentities, i.Fingerprint)
-		}
-	}
-	return keyIdentities, nil
-}
-
-// Sign implements SigningMechanism.Sign
-func (m gpgSigningMechanism) Sign(input []byte, keyIdentity string) ([]byte, error) {
-	key, err := m.ctx.GetKey(keyIdentity, true)
-	if err != nil {
-		return nil, err
-	}
-	inputData, err := gpgme.NewDataBytes(input)
-	if err != nil {
-		return nil, err
-	}
-	var sigBuffer bytes.Buffer
-	sigData, err := gpgme.NewDataWriter(&sigBuffer)
-	if err != nil {
-		return nil, err
-	}
-	if err = m.ctx.Sign([]*gpgme.Key{key}, inputData, sigData, gpgme.SigModeNormal); err != nil {
-		return nil, err
-	}
-	return sigBuffer.Bytes(), nil
-}
-
-// Verify implements SigningMechanism.Verify
-func (m gpgSigningMechanism) Verify(unverifiedSignature []byte) (contents []byte, keyIdentity string, err error) {
-	signedBuffer := bytes.Buffer{}
-	signedData, err := gpgme.NewDataWriter(&signedBuffer)
-	if err != nil {
-		return nil, "", err
-	}
-	unverifiedSignatureData, err := gpgme.NewDataBytes(unverifiedSignature)
-	if err != nil {
-		return nil, "", err
-	}
-	_, sigs, err := m.ctx.Verify(unverifiedSignatureData, nil, signedData)
-	if err != nil {
-		return nil, "", err
-	}
-	if len(sigs) != 1 {
-		return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Unexpected GPG signature count %d", len(sigs))}
-	}
-	sig := sigs[0]
-	// This is sig.Summary == gpgme.SigSumValid except for key trust, which we handle ourselves
-	if sig.Status != nil || sig.Validity == gpgme.ValidityNever || sig.ValidityReason != nil || sig.WrongKeyUsage {
-		// FIXME: Better error reporting eventually
-		return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Invalid GPG signature: %#v", sig)}
-	}
-	return signedBuffer.Bytes(), sig.Fingerprint, nil
-}
-
-// UntrustedSignatureContents returns UNTRUSTED contents of the signature WITHOUT ANY VERIFICATION,
+// gpgUntrustedSignatureContents returns UNTRUSTED contents of the signature WITHOUT ANY VERIFICATION,
 // along with a short identifier of the key used for signing.
 // WARNING: The short key identifier (which correponds to "Key ID" for OpenPGP keys)
 // is NOT the same as a "key identity" used in other calls ot this interface, and
 // the values may have no recognizable relationship if the public key is not available.
-func (m gpgSigningMechanism) UntrustedSignatureContents(untrustedSignature []byte) (untrustedContents []byte, shortKeyIdentifier string, err error) {
+func gpgUntrustedSignatureContents(untrustedSignature []byte) (untrustedContents []byte, shortKeyIdentifier string, err error) {
 	// This uses the Golang-native OpenPGP implementation instead of gpgme because we are not doing any cryptography.
 	md, err := openpgp.ReadMessage(bytes.NewReader(untrustedSignature), openpgp.EntityList{}, nil, nil)
 	if err != nil {

vendor/github.com/containers/image/signature/mechanism_gpgme.go

@@ -0,0 +1,175 @@
+// +build !containers_image_openpgp
+
+package signature
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/mtrmac/gpgme"
+)
+
+// A GPG/OpenPGP signing mechanism, implemented using gpgme.
+type gpgmeSigningMechanism struct {
+	ctx          *gpgme.Context
+	ephemeralDir string // If not "", a directory to be removed on Close()
+}
+
+// newGPGSigningMechanismInDirectory returns a new GPG/OpenPGP signing mechanism, using optionalDir if not empty.
+// The caller must call .Close() on the returned SigningMechanism.
+func newGPGSigningMechanismInDirectory(optionalDir string) (SigningMechanism, error) {
+	ctx, err := newGPGMEContext(optionalDir)
+	if err != nil {
+		return nil, err
+	}
+	return &gpgmeSigningMechanism{
+		ctx:          ctx,
+		ephemeralDir: "",
+	}, nil
+}
+
+// newEphemeralGPGSigningMechanism returns a new GPG/OpenPGP signing mechanism which
+// recognizes _only_ public keys from the supplied blob, and returns the identities
+// of these keys.
+// The caller must call .Close() on the returned SigningMechanism.
+func newEphemeralGPGSigningMechanism(blob []byte) (SigningMechanism, []string, error) {
+	dir, err := ioutil.TempDir("", "containers-ephemeral-gpg-")
+	if err != nil {
+		return nil, nil, err
+	}
+	removeDir := true
+	defer func() {
+		if removeDir {
+			os.RemoveAll(dir)
+		}
+	}()
+	ctx, err := newGPGMEContext(dir)
+	if err != nil {
+		return nil, nil, err
+	}
+	mech := &gpgmeSigningMechanism{
+		ctx:          ctx,
+		ephemeralDir: dir,
+	}
+	keyIdentities, err := mech.importKeysFromBytes(blob)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	removeDir = false
+	return mech, keyIdentities, nil
+}
+
+// newGPGMEContext returns a new *gpgme.Context, using optionalDir if not empty.
+func newGPGMEContext(optionalDir string) (*gpgme.Context, error) {
+	ctx, err := gpgme.New()
+	if err != nil {
+		return nil, err
+	}
+	if err = ctx.SetProtocol(gpgme.ProtocolOpenPGP); err != nil {
+		return nil, err
+	}
+	if optionalDir != "" {
+		err := ctx.SetEngineInfo(gpgme.ProtocolOpenPGP, "", optionalDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+	ctx.SetArmor(false)
+	ctx.SetTextMode(false)
+	return ctx, nil
+}
+
+func (m *gpgmeSigningMechanism) Close() error {
+	if m.ephemeralDir != "" {
+		os.RemoveAll(m.ephemeralDir) // Ignore an error, if any
+	}
+	return nil
+}
+
+// importKeysFromBytes imports public keys from the supplied blob and returns their identities.
+// The blob is assumed to have an appropriate format (the caller is expected to know which one).
+// NOTE: This may modify long-term state (e.g. key storage in a directory underlying the mechanism);
+// but we do not make this public, it can only be used through newEphemeralGPGSigningMechanism.
+func (m *gpgmeSigningMechanism) importKeysFromBytes(blob []byte) ([]string, error) {
+	inputData, err := gpgme.NewDataBytes(blob)
+	if err != nil {
+		return nil, err
+	}
+	res, err := m.ctx.Import(inputData)
+	if err != nil {
+		return nil, err
+	}
+	keyIdentities := []string{}
+	for _, i := range res.Imports {
+		if i.Result == nil {
+			keyIdentities = append(keyIdentities, i.Fingerprint)
+		}
+	}
+	return keyIdentities, nil
+}
+
+// SupportsSigning returns nil if the mechanism supports signing, or a SigningNotSupportedError.
+func (m *gpgmeSigningMechanism) SupportsSigning() error {
+	return nil
+}
+
+// Sign creates a (non-detached) signature of input using keyIdentity.
+// Fails with a SigningNotSupportedError if the mechanism does not support signing.
+func (m *gpgmeSigningMechanism) Sign(input []byte, keyIdentity string) ([]byte, error) {
+	key, err := m.ctx.GetKey(keyIdentity, true)
+	if err != nil {
+		return nil, err
+	}
+	inputData, err := gpgme.NewDataBytes(input)
+	if err != nil {
+		return nil, err
+	}
+	var sigBuffer bytes.Buffer
+	sigData, err := gpgme.NewDataWriter(&sigBuffer)
+	if err != nil {
+		return nil, err
+	}
+	if err = m.ctx.Sign([]*gpgme.Key{key}, inputData, sigData, gpgme.SigModeNormal); err != nil {
+		return nil, err
+	}
+	return sigBuffer.Bytes(), nil
+}
+
+// Verify parses unverifiedSignature and returns the content and the signer's identity
+func (m gpgmeSigningMechanism) Verify(unverifiedSignature []byte) (contents []byte, keyIdentity string, err error) {
+	signedBuffer := bytes.Buffer{}
+	signedData, err := gpgme.NewDataWriter(&signedBuffer)
+	if err != nil {
+		return nil, "", err
+	}
+	unverifiedSignatureData, err := gpgme.NewDataBytes(unverifiedSignature)
+	if err != nil {
+		return nil, "", err
+	}
+	_, sigs, err := m.ctx.Verify(unverifiedSignatureData, nil, signedData)
+	if err != nil {
+		return nil, "", err
+	}
+	if len(sigs) != 1 {
+		return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Unexpected GPG signature count %d", len(sigs))}
+	}
+	sig := sigs[0]
+	// This is sig.Summary == gpgme.SigSumValid except for key trust, which we handle ourselves
+	if sig.Status != nil || sig.Validity == gpgme.ValidityNever || sig.ValidityReason != nil || sig.WrongKeyUsage {
+		// FIXME: Better error reporting eventually
+		return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Invalid GPG signature: %#v", sig)}
+	}
+	return signedBuffer.Bytes(), sig.Fingerprint, nil
+}
+
+// UntrustedSignatureContents returns UNTRUSTED contents of the signature WITHOUT ANY VERIFICATION,
+// along with a short identifier of the key used for signing.
+// WARNING: The short key identifier (which correponds to "Key ID" for OpenPGP keys)
+// is NOT the same as a "key identity" used in other calls ot this interface, and
+// the values may have no recognizable relationship if the public key is not available.
+func (m gpgmeSigningMechanism) UntrustedSignatureContents(untrustedSignature []byte) (untrustedContents []byte, shortKeyIdentifier string, err error) {
+	return gpgUntrustedSignatureContents(untrustedSignature)
+}

vendor/github.com/containers/image/signature/mechanism_openpgp.go

@@ -0,0 +1,153 @@
+// +build containers_image_openpgp
+
+package signature
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/containers/storage/pkg/homedir"
+	"golang.org/x/crypto/openpgp"
+)
+
+// A GPG/OpenPGP signing mechanism, implemented using x/crypto/openpgp.
+type openpgpSigningMechanism struct {
+	keyring openpgp.EntityList
+}
+
+// newGPGSigningMechanismInDirectory returns a new GPG/OpenPGP signing mechanism, using optionalDir if not empty.
+// The caller must call .Close() on the returned SigningMechanism.
+func newGPGSigningMechanismInDirectory(optionalDir string) (SigningMechanism, error) {
+	m := &openpgpSigningMechanism{
+		keyring: openpgp.EntityList{},
+	}
+
+	gpgHome := optionalDir
+	if gpgHome == "" {
+		gpgHome = os.Getenv("GNUPGHOME")
+		if gpgHome == "" {
+			gpgHome = path.Join(homedir.Get(), ".gnupg")
+		}
+	}
+
+	pubring, err := ioutil.ReadFile(path.Join(gpgHome, "pubring.gpg"))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+	} else {
+		_, err := m.importKeysFromBytes(pubring)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return m, nil
+}
+
+// newEphemeralGPGSigningMechanism returns a new GPG/OpenPGP signing mechanism which
+// recognizes _only_ public keys from the supplied blob, and returns the identities
+// of these keys.
+// The caller must call .Close() on the returned SigningMechanism.
+func newEphemeralGPGSigningMechanism(blob []byte) (SigningMechanism, []string, error) {
+	m := &openpgpSigningMechanism{
+		keyring: openpgp.EntityList{},
+	}
+	keyIdentities, err := m.importKeysFromBytes(blob)
+	if err != nil {
+		return nil, nil, err
+	}
+	return m, keyIdentities, nil
+}
+
+func (m *openpgpSigningMechanism) Close() error {
+	return nil
+}
+
+// importKeysFromBytes imports public keys from the supplied blob and returns their identities.
+// The blob is assumed to have an appropriate format (the caller is expected to know which one).
+func (m *openpgpSigningMechanism) importKeysFromBytes(blob []byte) ([]string, error) {
+	keyring, err := openpgp.ReadKeyRing(bytes.NewReader(blob))
+	if err != nil {
+		k, e2 := openpgp.ReadArmoredKeyRing(bytes.NewReader(blob))
+		if e2 != nil {
+			return nil, err // The original error  -- FIXME: is this better?
+		}
+		keyring = k
+	}
+
+	keyIdentities := []string{}
+	for _, entity := range keyring {
+		if entity.PrimaryKey == nil {
+			// Coverage: This should never happen, openpgp.ReadEntity fails with a
+			// openpgp.errors.StructuralError instead of returning an entity with this
+			// field set to nil.
+			continue
+		}
+		// Uppercase the fingerprint to be compatible with gpgme
+		keyIdentities = append(keyIdentities, strings.ToUpper(fmt.Sprintf("%x", entity.PrimaryKey.Fingerprint)))
+		m.keyring = append(m.keyring, entity)
+	}
+	return keyIdentities, nil
+}
+
+// SupportsSigning returns nil if the mechanism supports signing, or a SigningNotSupportedError.
+func (m *openpgpSigningMechanism) SupportsSigning() error {
+	return SigningNotSupportedError("signing is not supported in github.com/containers/image built with the containers_image_openpgp build tag")
+}
+
+// Sign creates a (non-detached) signature of input using keyIdentity.
+// Fails with a SigningNotSupportedError if the mechanism does not support signing.
+func (m *openpgpSigningMechanism) Sign(input []byte, keyIdentity string) ([]byte, error) {
+	return nil, SigningNotSupportedError("signing is not supported in github.com/containers/image built with the containers_image_openpgp build tag")
+}
+
+// Verify parses unverifiedSignature and returns the content and the signer's identity
+func (m *openpgpSigningMechanism) Verify(unverifiedSignature []byte) (contents []byte, keyIdentity string, err error) {
+	md, err := openpgp.ReadMessage(bytes.NewReader(unverifiedSignature), m.keyring, nil, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	if !md.IsSigned {
+		return nil, "", errors.New("not signed")
+	}
+	content, err := ioutil.ReadAll(md.UnverifiedBody)
+	if err != nil {
+		// Coverage: md.UnverifiedBody.Read only fails if the body is encrypted
+		// (and possibly also signed, but it _must_ be encrypted) and the signing
+		// “modification detection code” detects a mismatch. But in that case,
+		// we would expect the signature verification to fail as well, and that is checked
+		// first.  Besides, we are not supplying any decryption keys, so we really
+		// can never reach this “encrypted data MDC mismatch” path.
+		return nil, "", err
+	}
+	if md.SignatureError != nil {
+		return nil, "", fmt.Errorf("signature error: %v", md.SignatureError)
+	}
+	if md.SignedBy == nil {
+		return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Invalid GPG signature: %#v", md.Signature)}
+	}
+	if md.Signature.SigLifetimeSecs != nil {
+		expiry := md.Signature.CreationTime.Add(time.Duration(*md.Signature.SigLifetimeSecs) * time.Second)
+		if time.Now().After(expiry) {
+			return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Signature expired on %s", expiry)}
+		}
+	}
+
+	// Uppercase the fingerprint to be compatible with gpgme
+	return content, strings.ToUpper(fmt.Sprintf("%x", md.SignedBy.PublicKey.Fingerprint)), nil
+}
+
+// UntrustedSignatureContents returns UNTRUSTED contents of the signature WITHOUT ANY VERIFICATION,
+// along with a short identifier of the key used for signing.
+// WARNING: The short key identifier (which correponds to "Key ID" for OpenPGP keys)
+// is NOT the same as a "key identity" used in other calls ot this interface, and
+// the values may have no recognizable relationship if the public key is not available.
+func (m openpgpSigningMechanism) UntrustedSignatureContents(untrustedSignature []byte) (untrustedContents []byte, shortKeyIdentifier string, err error) {
+	return gpgUntrustedSignatureContents(untrustedSignature)
+}

vendor/github.com/containers/image/signature/policy_config.go

@@ -255,13 +255,8 @@ var _ json.Unmarshaler = (*prInsecureAcceptAnything)(nil)
 func (pr *prInsecureAcceptAnything) UnmarshalJSON(data []byte) error {
 	*pr = prInsecureAcceptAnything{}
 	var tmp prInsecureAcceptAnything
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type": &tmp.Type,
 	}); err != nil {
 		return err
 	}
@@ -290,13 +285,8 @@ var _ json.Unmarshaler = (*prReject)(nil)
 func (pr *prReject) UnmarshalJSON(data []byte) error {
 	*pr = prReject{}
 	var tmp prReject
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type": &tmp.Type,
 	}); err != nil {
 		return err
 	}
@@ -465,15 +455,9 @@ func (pr *prSignedBaseLayer) UnmarshalJSON(data []byte) error {
 	*pr = prSignedBaseLayer{}
 	var tmp prSignedBaseLayer
 	var baseLayerIdentity json.RawMessage
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		case "baseLayerIdentity":
-			return &baseLayerIdentity
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type":              &tmp.Type,
+		"baseLayerIdentity": &baseLayerIdentity,
 	}); err != nil {
 		return err
 	}
@@ -481,9 +465,6 @@ func (pr *prSignedBaseLayer) UnmarshalJSON(data []byte) error {
 	if tmp.Type != prTypeSignedBaseLayer {
 		return InvalidPolicyFormatError(fmt.Sprintf("Unexpected policy requirement type \"%s\"", tmp.Type))
 	}
-	if baseLayerIdentity == nil {
-		return InvalidPolicyFormatError(fmt.Sprintf("baseLayerIdentity not specified"))
-	}
 	bli, err := newPolicyReferenceMatchFromJSON(baseLayerIdentity)
 	if err != nil {
 		return err
@@ -541,13 +522,8 @@ var _ json.Unmarshaler = (*prmMatchExact)(nil)
 func (prm *prmMatchExact) UnmarshalJSON(data []byte) error {
 	*prm = prmMatchExact{}
 	var tmp prmMatchExact
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type": &tmp.Type,
 	}); err != nil {
 		return err
 	}
@@ -576,13 +552,8 @@ var _ json.Unmarshaler = (*prmMatchRepoDigestOrExact)(nil)
 func (prm *prmMatchRepoDigestOrExact) UnmarshalJSON(data []byte) error {
 	*prm = prmMatchRepoDigestOrExact{}
 	var tmp prmMatchRepoDigestOrExact
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type": &tmp.Type,
 	}); err != nil {
 		return err
 	}
@@ -611,13 +582,8 @@ var _ json.Unmarshaler = (*prmMatchRepository)(nil)
 func (prm *prmMatchRepository) UnmarshalJSON(data []byte) error {
 	*prm = prmMatchRepository{}
 	var tmp prmMatchRepository
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type": &tmp.Type,
 	}); err != nil {
 		return err
 	}
@@ -656,15 +622,9 @@ var _ json.Unmarshaler = (*prmExactReference)(nil)
 func (prm *prmExactReference) UnmarshalJSON(data []byte) error {
 	*prm = prmExactReference{}
 	var tmp prmExactReference
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		case "dockerReference":
-			return &tmp.DockerReference
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type":            &tmp.Type,
+		"dockerReference": &tmp.DockerReference,
 	}); err != nil {
 		return err
 	}
@@ -704,15 +664,9 @@ var _ json.Unmarshaler = (*prmExactRepository)(nil)
 func (prm *prmExactRepository) UnmarshalJSON(data []byte) error {
 	*prm = prmExactRepository{}
 	var tmp prmExactRepository
-	if err := paranoidUnmarshalJSONObject(data, func(key string) interface{} {
-		switch key {
-		case "type":
-			return &tmp.Type
-		case "dockerRepository":
-			return &tmp.DockerRepository
-		default:
-			return nil
-		}
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"type":             &tmp.Type,
+		"dockerRepository": &tmp.DockerRepository,
 	}); err != nil {
 		return err
 	}

vendor/github.com/containers/image/signature/policy_eval_signedby.go

@@ -5,7 +5,6 @@ package signature
 import (
 	"fmt"
 	"io/ioutil"
-	"os"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -42,20 +41,11 @@ func (pr *prSignedBy) isSignatureAuthorAccepted(image types.UnparsedImage, sig [
 	}
 
 	// FIXME: move this to per-context initialization
-	dir, err := ioutil.TempDir("", "skopeo-signedBy-")
-	if err != nil {
-		return sarRejected, nil, err
-	}
-	defer os.RemoveAll(dir)
-	mech, err := newGPGSigningMechanismInDirectory(dir)
-	if err != nil {
-		return sarRejected, nil, err
-	}
-
-	trustedIdentities, err := mech.ImportKeysFromBytes(data)
+	mech, trustedIdentities, err := NewEphemeralGPGSigningMechanism(data)
 	if err != nil {
 		return sarRejected, nil, err
 	}
+	defer mech.Close()
 	if len(trustedIdentities) == 0 {
 		return sarRejected, nil, PolicyRequirementError("No public keys imported")
 	}

vendor/github.com/containers/image/signature/signature.go

@@ -120,78 +120,69 @@ func (s *untrustedSignature) UnmarshalJSON(data []byte) error {
 // strictUnmarshalJSON is UnmarshalJSON, except that it may return the internal jsonFormatError error type.
 // Splitting it into a separate function allows us to do the jsonFormatError → InvalidSignatureError in a single place, the caller.
 func (s *untrustedSignature) strictUnmarshalJSON(data []byte) error {
-	var untyped interface{}
-	if err := json.Unmarshal(data, &untyped); err != nil {
-		return err
-	}
-	o, ok := untyped.(map[string]interface{})
-	if !ok {
-		return InvalidSignatureError{msg: "Invalid signature format"}
-	}
-	if err := validateExactMapKeys(o, "critical", "optional"); err != nil {
+	var critical, optional json.RawMessage
+	if err := paranoidUnmarshalJSONObjectExactFields(data, map[string]interface{}{
+		"critical": &critical,
+		"optional": &optional,
+	}); err != nil {
 		return err
 	}
 
-	c, err := mapField(o, "critical")
-	if err != nil {
-		return err
-	}
-	if err := validateExactMapKeys(c, "type", "image", "identity"); err != nil {
-		return err
-	}
-
-	optional, err := mapField(o, "optional")
-	if err != nil {
-		return err
-	}
-	if _, ok := optional["creator"]; ok {
-		creatorID, err := stringField(optional, "creator")
-		if err != nil {
-			return err
+	var creatorID string
+	var timestamp float64
+	var gotCreatorID, gotTimestamp = false, false
+	if err := paranoidUnmarshalJSONObject(optional, func(key string) interface{} {
+		switch key {
+		case "creator":
+			gotCreatorID = true
+			return &creatorID
+		case "timestamp":
+			gotTimestamp = true
+			return &timestamp
+		default:
+			var ignore interface{}
+			return &ignore
 		}
+	}); err != nil {
+		return err
+	}
+	if gotCreatorID {
 		s.UntrustedCreatorID = &creatorID
 	}
-	if _, ok := optional["timestamp"]; ok {
-		timestamp, err := int64Field(optional, "timestamp")
-		if err != nil {
-			return err
+	if gotTimestamp {
+		intTimestamp := int64(timestamp)
+		if float64(intTimestamp) != timestamp {
+			return InvalidSignatureError{msg: "Field optional.timestamp is not is not an integer"}
 		}
-		s.UntrustedTimestamp = &timestamp
+		s.UntrustedTimestamp = &intTimestamp
 	}
 
-	t, err := stringField(c, "type")
-	if err != nil {
+	var t string
+	var image, identity json.RawMessage
+	if err := paranoidUnmarshalJSONObjectExactFields(critical, map[string]interface{}{
+		"type":     &t,
+		"image":    &image,
+		"identity": &identity,
+	}); err != nil {
 		return err
 	}
 	if t != signatureType {
 		return InvalidSignatureError{msg: fmt.Sprintf("Unrecognized signature type %s", t)}
 	}
 
-	image, err := mapField(c, "image")
-	if err != nil {
-		return err
-	}
-	if err := validateExactMapKeys(image, "docker-manifest-digest"); err != nil {
-		return err
-	}
-	digestString, err := stringField(image, "docker-manifest-digest")
-	if err != nil {
+	var digestString string
+	if err := paranoidUnmarshalJSONObjectExactFields(image, map[string]interface{}{
+		"docker-manifest-digest": &digestString,
+	}); err != nil {
 		return err
 	}
 	s.UntrustedDockerManifestDigest = digest.Digest(digestString)
 
-	identity, err := mapField(c, "identity")
-	if err != nil {
+	if err := paranoidUnmarshalJSONObjectExactFields(identity, map[string]interface{}{
+		"docker-reference": &s.UntrustedDockerReference,
+	}); err != nil {
 		return err
 	}
-	if err := validateExactMapKeys(identity, "docker-reference"); err != nil {
-		return err
-	}
-	reference, err := stringField(identity, "docker-reference")
-	if err != nil {
-		return err
-	}
-	s.UntrustedDockerReference = reference
 
 	return nil
 }
@@ -261,10 +252,11 @@ func verifyAndExtractSignature(mech SigningMechanism, unverifiedSignature []byte
 // (including things like “✅ Verified by $authority”)
 func GetUntrustedSignatureInformationWithoutVerifying(untrustedSignatureBytes []byte) (*UntrustedSignatureInformation, error) {
 	// NOTE: This should eventualy do format autodetection.
-	mech, err := NewGPGSigningMechanism()
+	mech, _, err := NewEphemeralGPGSigningMechanism([]byte{})
 	if err != nil {
 		return nil, err
 	}
+	defer mech.Close()
 
 	untrustedContents, shortKeyIdentifier, err := mech.UntrustedSignatureContents(untrustedSignatureBytes)
 	if err != nil {

vendor/github.com/containers/storage/README.md

@@ -2,9 +2,7 @@
 layers, container images, and containers.  An `oci-storage` CLI wrapper is also
 included for manual and scripting use.
 
-To build the CLI wrapper, use 'make build-binary', optionally passing
-'AUTO_GOPATH=1' as an additional argument to avoid having to set $GOPATH
-manually.  For information on other recognized targets, run 'make help'.
+To build the CLI wrapper, use 'make build-binary'.
 
 Operations which use VMs expect to launch them using 'vagrant', defaulting to
 using its 'libvirt' provider.  The boxes used are also available for the

vendor/github.com/containers/storage/drivers/aufs/aufs.go

@@ -45,8 +45,8 @@ import (
 	mountpk "github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/stringid"
 
-	"github.com/opencontainers/runc/libcontainer/label"
 	rsystem "github.com/opencontainers/runc/libcontainer/system"
+	"github.com/opencontainers/selinux/go-selinux/label"
 )
 
 var (

vendor/github.com/containers/storage/drivers/btrfs/btrfs.go

@@ -28,7 +28,7 @@ import (
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/docker/go-units"
-	"github.com/opencontainers/runc/libcontainer/label"
+	"github.com/opencontainers/selinux/go-selinux/label"
 )
 
 func init() {

vendor/github.com/containers/storage/drivers/devmapper/deviceset.go

@@ -31,7 +31,7 @@ import (
 	"github.com/containers/storage/storageversion"
 	"github.com/docker/go-units"
 
-	"github.com/opencontainers/runc/libcontainer/label"
+	"github.com/opencontainers/selinux/go-selinux/label"
 )
 
 var (
@@ -2569,7 +2569,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 			devices.metaDataLoopbackSize = size
 		case "dm.fs":
 			if val != "ext4" && val != "xfs" {
-				return nil, fmt.Errorf("devmapper: Unsupported filesystem %s\n", val)
+				return nil, fmt.Errorf("devmapper: Unsupported filesystem %s", val)
 			}
 			devices.filesystem = val
 		case "dm.mkfsarg":
@@ -2631,9 +2631,9 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 			devices.minFreeSpacePercent = uint32(minFreeSpacePercent)
 		default:
 			if nthOption > len(defaults) {
-				return nil, fmt.Errorf("devmapper: Unknown option %s\n", key)
+				return nil, fmt.Errorf("devmapper: Unknown option %s", key)
 			}
-			logrus.Errorf("devmapper: Unknown option %s, ignoring\n", key)
+			logrus.Errorf("devmapper: Unknown option %s, ignoring", key)
 		}
 		settings[key] = val
 	}

vendor/github.com/containers/storage/drivers/driver_linux.go

@@ -53,11 +53,11 @@ const (
 var (
 	// Slice of drivers that should be used in an order
 	priority = []string{
+		"overlay2",
+		"devicemapper",
 		"aufs",
 		"btrfs",
 		"zfs",
-		"devicemapper",
-		"overlay",
 		"vfs",
 	}
 
@@ -72,6 +72,7 @@ var (
 		FsMagicJffs2Fs:     "jffs2",
 		FsMagicJfs:         "jfs",
 		FsMagicNfsFs:       "nfs",
+		FsMagicOverlay:     "overlay",
 		FsMagicRAMFs:       "ramfs",
 		FsMagicReiserFs:    "reiserfs",
 		FsMagicSmbFs:       "smb",

vendor/github.com/containers/storage/drivers/overlay/copy.go

@@ -1,169 +0,0 @@
-// +build linux
-
-package overlay
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"syscall"
-	"time"
-
-	"github.com/containers/storage/pkg/pools"
-	"github.com/containers/storage/pkg/system"
-)
-
-type copyFlags int
-
-const (
-	copyHardlink copyFlags = 1 << iota
-)
-
-func copyRegular(srcPath, dstPath string, mode os.FileMode) error {
-	srcFile, err := os.Open(srcPath)
-	if err != nil {
-		return err
-	}
-	defer srcFile.Close()
-
-	dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE, mode)
-	if err != nil {
-		return err
-	}
-	defer dstFile.Close()
-
-	_, err = pools.Copy(dstFile, srcFile)
-
-	return err
-}
-
-func copyXattr(srcPath, dstPath, attr string) error {
-	data, err := system.Lgetxattr(srcPath, attr)
-	if err != nil {
-		return err
-	}
-	if data != nil {
-		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func copyDir(srcDir, dstDir string, flags copyFlags) error {
-	err := filepath.Walk(srcDir, func(srcPath string, f os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Rebase path
-		relPath, err := filepath.Rel(srcDir, srcPath)
-		if err != nil {
-			return err
-		}
-
-		dstPath := filepath.Join(dstDir, relPath)
-		if err != nil {
-			return err
-		}
-
-		stat, ok := f.Sys().(*syscall.Stat_t)
-		if !ok {
-			return fmt.Errorf("Unable to get raw syscall.Stat_t data for %s", srcPath)
-		}
-
-		isHardlink := false
-
-		switch f.Mode() & os.ModeType {
-		case 0: // Regular file
-			if flags&copyHardlink != 0 {
-				isHardlink = true
-				if err := os.Link(srcPath, dstPath); err != nil {
-					return err
-				}
-			} else {
-				if err := copyRegular(srcPath, dstPath, f.Mode()); err != nil {
-					return err
-				}
-			}
-
-		case os.ModeDir:
-			if err := os.Mkdir(dstPath, f.Mode()); err != nil && !os.IsExist(err) {
-				return err
-			}
-
-		case os.ModeSymlink:
-			link, err := os.Readlink(srcPath)
-			if err != nil {
-				return err
-			}
-
-			if err := os.Symlink(link, dstPath); err != nil {
-				return err
-			}
-
-		case os.ModeNamedPipe:
-			fallthrough
-		case os.ModeSocket:
-			if err := syscall.Mkfifo(dstPath, stat.Mode); err != nil {
-				return err
-			}
-
-		case os.ModeDevice:
-			if err := syscall.Mknod(dstPath, stat.Mode, int(stat.Rdev)); err != nil {
-				return err
-			}
-
-		default:
-			return fmt.Errorf("Unknown file type for %s\n", srcPath)
-		}
-
-		// Everything below is copying metadata from src to dst. All this metadata
-		// already shares an inode for hardlinks.
-		if isHardlink {
-			return nil
-		}
-
-		if err := os.Lchown(dstPath, int(stat.Uid), int(stat.Gid)); err != nil {
-			return err
-		}
-
-		if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
-			return err
-		}
-
-		// We need to copy this attribute if it appears in an overlay upper layer, as
-		// this function is used to copy those. It is set by overlay if a directory
-		// is removed and then re-created and should not inherit anything from the
-		// same dir in the lower dir.
-		if err := copyXattr(srcPath, dstPath, "trusted.overlay.opaque"); err != nil {
-			return err
-		}
-
-		isSymlink := f.Mode()&os.ModeSymlink != 0
-
-		// There is no LChmod, so ignore mode for symlink. Also, this
-		// must happen after chown, as that can modify the file mode
-		if !isSymlink {
-			if err := os.Chmod(dstPath, f.Mode()); err != nil {
-				return err
-			}
-		}
-
-		// system.Chtimes doesn't support a NOFOLLOW flag atm
-		if !isSymlink {
-			aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec))
-			mTime := time.Unix(int64(stat.Mtim.Sec), int64(stat.Mtim.Nsec))
-			if err := system.Chtimes(dstPath, aTime, mTime); err != nil {
-				return err
-			}
-		} else {
-			ts := []syscall.Timespec{stat.Atim, stat.Mtim}
-			if err := system.LUtimesNano(dstPath, ts); err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	return err
-}

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -1,450 +0,0 @@
-// +build linux
-
-package overlay
-
-import (
-	"bufio"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path"
-	"syscall"
-
-	"github.com/Sirupsen/logrus"
-
-	"github.com/containers/storage/drivers"
-	"github.com/containers/storage/pkg/archive"
-	"github.com/containers/storage/pkg/idtools"
-
-	"github.com/containers/storage/pkg/mount"
-	"github.com/opencontainers/runc/libcontainer/label"
-)
-
-// This is a small wrapper over the NaiveDiffWriter that lets us have a custom
-// implementation of ApplyDiff()
-
-var (
-	// ErrApplyDiffFallback is returned to indicate that a normal ApplyDiff is applied as a fallback from Naive diff writer.
-	ErrApplyDiffFallback = fmt.Errorf("Fall back to normal ApplyDiff")
-	backingFs            = "<unknown>"
-)
-
-// ApplyDiffProtoDriver wraps the ProtoDriver by extending the interface with ApplyDiff method.
-type ApplyDiffProtoDriver interface {
-	graphdriver.ProtoDriver
-	// ApplyDiff writes the diff to the archive for the given id and parent id.
-	// It returns the size in bytes written if successful, an error ErrApplyDiffFallback is returned otherwise.
-	ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error)
-}
-
-type naiveDiffDriverWithApply struct {
-	graphdriver.Driver
-	applyDiff ApplyDiffProtoDriver
-}
-
-// NaiveDiffDriverWithApply returns a NaiveDiff driver with custom ApplyDiff.
-func NaiveDiffDriverWithApply(driver ApplyDiffProtoDriver, uidMaps, gidMaps []idtools.IDMap) graphdriver.Driver {
-	return &naiveDiffDriverWithApply{
-		Driver:    graphdriver.NewNaiveDiffDriver(driver, uidMaps, gidMaps),
-		applyDiff: driver,
-	}
-}
-
-// ApplyDiff creates a diff layer with either the NaiveDiffDriver or with a fallback.
-func (d *naiveDiffDriverWithApply) ApplyDiff(id, parent string, diff archive.Reader) (int64, error) {
-	b, err := d.applyDiff.ApplyDiff(id, parent, diff)
-	if err == ErrApplyDiffFallback {
-		return d.Driver.ApplyDiff(id, parent, diff)
-	}
-	return b, err
-}
-
-// This backend uses the overlay union filesystem for containers
-// plus hard link file sharing for images.
-
-// Each container/image can have a "root" subdirectory which is a plain
-// filesystem hierarchy, or they can use overlay.
-
-// If they use overlay there is a "upper" directory and a "lower-id"
-// file, as well as "merged" and "work" directories. The "upper"
-// directory has the upper layer of the overlay, and "lower-id" contains
-// the id of the parent whose "root" directory shall be used as the lower
-// layer in the overlay. The overlay itself is mounted in the "merged"
-// directory, and the "work" dir is needed for overlay to work.
-
-// When an overlay layer is created there are two cases, either the
-// parent has a "root" dir, then we start out with an empty "upper"
-// directory overlaid on the parents root. This is typically the
-// case with the init layer of a container which is based on an image.
-// If there is no "root" in the parent, we inherit the lower-id from
-// the parent and start by making a copy in the parent's "upper" dir.
-// This is typically the case for a container layer which copies
-// its parent -init upper layer.
-
-// Additionally we also have a custom implementation of ApplyLayer
-// which makes a recursive copy of the parent "root" layer using
-// hardlinks to share file data, and then applies the layer on top
-// of that. This means all child images share file (but not directory)
-// data with the parent.
-
-// Driver contains information about the home directory and the list of active mounts that are created using this driver.
-type Driver struct {
-	home    string
-	uidMaps []idtools.IDMap
-	gidMaps []idtools.IDMap
-	ctr     *graphdriver.RefCounter
-}
-
-func init() {
-	graphdriver.Register("overlay", Init)
-}
-
-// Init returns the NaiveDiffDriver, a native diff driver for overlay filesystem.
-// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
-// If an overlay filesystem is not supported over an existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
-func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
-
-	if err := supportsOverlay(); err != nil {
-		return nil, graphdriver.ErrNotSupported
-	}
-
-	fsMagic, err := graphdriver.GetFSMagic(home)
-	if err != nil {
-		return nil, err
-	}
-	if fsName, ok := graphdriver.FsNames[fsMagic]; ok {
-		backingFs = fsName
-	}
-
-	switch fsMagic {
-	case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs, graphdriver.FsMagicEcryptfs:
-		logrus.Errorf("'overlay' is not supported over %s", backingFs)
-		return nil, graphdriver.ErrIncompatibleFS
-	}
-
-	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
-	if err != nil {
-		return nil, err
-	}
-	// Create the driver home dir
-	if err := idtools.MkdirAllAs(home, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
-		return nil, err
-	}
-
-	if err := mount.MakePrivate(home); err != nil {
-		return nil, err
-	}
-
-	d := &Driver{
-		home:    home,
-		uidMaps: uidMaps,
-		gidMaps: gidMaps,
-		ctr:     graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)),
-	}
-
-	return NaiveDiffDriverWithApply(d, uidMaps, gidMaps), nil
-}
-
-func supportsOverlay() error {
-	// We can try to modprobe overlay first before looking at
-	// proc/filesystems for when overlay is supported
-	exec.Command("modprobe", "overlay").Run()
-
-	f, err := os.Open("/proc/filesystems")
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		if s.Text() == "nodev\toverlay" {
-			return nil
-		}
-	}
-	logrus.Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
-	return graphdriver.ErrNotSupported
-}
-
-func (d *Driver) String() string {
-	return "overlay"
-}
-
-// Status returns current driver information in a two dimensional string array.
-// Output contains "Backing Filesystem" used in this implementation.
-func (d *Driver) Status() [][2]string {
-	return [][2]string{
-		{"Backing Filesystem", backingFs},
-	}
-}
-
-// GetMetadata returns meta data about the overlay driver such as root, LowerDir, UpperDir, WorkDir and MergeDir used to store data.
-func (d *Driver) GetMetadata(id string) (map[string]string, error) {
-	dir := d.dir(id)
-	if _, err := os.Stat(dir); err != nil {
-		return nil, err
-	}
-
-	metadata := make(map[string]string)
-
-	// If id has a root, it is an image
-	rootDir := path.Join(dir, "root")
-	if _, err := os.Stat(rootDir); err == nil {
-		metadata["RootDir"] = rootDir
-		return metadata, nil
-	}
-
-	lowerID, err := ioutil.ReadFile(path.Join(dir, "lower-id"))
-	if err != nil {
-		return nil, err
-	}
-
-	metadata["LowerDir"] = path.Join(d.dir(string(lowerID)), "root")
-	metadata["UpperDir"] = path.Join(dir, "upper")
-	metadata["WorkDir"] = path.Join(dir, "work")
-	metadata["MergedDir"] = path.Join(dir, "merged")
-
-	return metadata, nil
-}
-
-// Cleanup any state created by overlay which should be cleaned when daemon
-// is being shutdown. For now, we just have to unmount the bind mounted
-// we had created.
-func (d *Driver) Cleanup() error {
-	return mount.Unmount(d.home)
-}
-
-// CreateReadWrite creates a layer that is writable for use as a container
-// file system.
-func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
-	return d.Create(id, parent, mountLabel, storageOpt)
-}
-
-// Create is used to create the upper, lower, and merge directories required for overlay fs for a given id.
-// The parent filesystem is used to configure these directories for the overlay.
-func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) (retErr error) {
-
-	if len(storageOpt) != 0 {
-		return fmt.Errorf("--storage-opt is not supported for overlay")
-	}
-
-	dir := d.dir(id)
-
-	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
-	if err != nil {
-		return err
-	}
-	if err := idtools.MkdirAllAs(path.Dir(dir), 0700, rootUID, rootGID); err != nil {
-		return err
-	}
-	if err := idtools.MkdirAs(dir, 0700, rootUID, rootGID); err != nil {
-		return err
-	}
-
-	defer func() {
-		// Clean up on failure
-		if retErr != nil {
-			os.RemoveAll(dir)
-		}
-	}()
-
-	// Toplevel images are just a "root" dir
-	if parent == "" {
-		if err := idtools.MkdirAs(path.Join(dir, "root"), 0755, rootUID, rootGID); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	parentDir := d.dir(parent)
-
-	// Ensure parent exists
-	if _, err := os.Lstat(parentDir); err != nil {
-		return err
-	}
-
-	// If parent has a root, just do an overlay to it
-	parentRoot := path.Join(parentDir, "root")
-
-	if s, err := os.Lstat(parentRoot); err == nil {
-		if err := idtools.MkdirAs(path.Join(dir, "upper"), s.Mode(), rootUID, rootGID); err != nil {
-			return err
-		}
-		if err := idtools.MkdirAs(path.Join(dir, "work"), 0700, rootUID, rootGID); err != nil {
-			return err
-		}
-		if err := idtools.MkdirAs(path.Join(dir, "merged"), 0700, rootUID, rootGID); err != nil {
-			return err
-		}
-		if err := ioutil.WriteFile(path.Join(dir, "lower-id"), []byte(parent), 0666); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	// Otherwise, copy the upper and the lower-id from the parent
-
-	lowerID, err := ioutil.ReadFile(path.Join(parentDir, "lower-id"))
-	if err != nil {
-		return err
-	}
-
-	if err := ioutil.WriteFile(path.Join(dir, "lower-id"), lowerID, 0666); err != nil {
-		return err
-	}
-
-	parentUpperDir := path.Join(parentDir, "upper")
-	s, err := os.Lstat(parentUpperDir)
-	if err != nil {
-		return err
-	}
-
-	upperDir := path.Join(dir, "upper")
-	if err := idtools.MkdirAs(upperDir, s.Mode(), rootUID, rootGID); err != nil {
-		return err
-	}
-	if err := idtools.MkdirAs(path.Join(dir, "work"), 0700, rootUID, rootGID); err != nil {
-		return err
-	}
-	if err := idtools.MkdirAs(path.Join(dir, "merged"), 0700, rootUID, rootGID); err != nil {
-		return err
-	}
-
-	return copyDir(parentUpperDir, upperDir, 0)
-}
-
-func (d *Driver) dir(id string) string {
-	return path.Join(d.home, id)
-}
-
-// Remove cleans the directories that are created for this id.
-func (d *Driver) Remove(id string) error {
-	if err := os.RemoveAll(d.dir(id)); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	return nil
-}
-
-// Get creates and mounts the required file system for the given id and returns the mount path.
-func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
-	dir := d.dir(id)
-	if _, err := os.Stat(dir); err != nil {
-		return "", err
-	}
-	// If id has a root, just return it
-	rootDir := path.Join(dir, "root")
-	if _, err := os.Stat(rootDir); err == nil {
-		return rootDir, nil
-	}
-	mergedDir := path.Join(dir, "merged")
-	if count := d.ctr.Increment(mergedDir); count > 1 {
-		return mergedDir, nil
-	}
-	defer func() {
-		if err != nil {
-			if c := d.ctr.Decrement(mergedDir); c <= 0 {
-				syscall.Unmount(mergedDir, 0)
-			}
-		}
-	}()
-	lowerID, err := ioutil.ReadFile(path.Join(dir, "lower-id"))
-	if err != nil {
-		return "", err
-	}
-	var (
-		lowerDir = path.Join(d.dir(string(lowerID)), "root")
-		upperDir = path.Join(dir, "upper")
-		workDir  = path.Join(dir, "work")
-		opts     = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDir, upperDir, workDir)
-	)
-	if err := syscall.Mount("overlay", mergedDir, "overlay", 0, label.FormatMountLabel(opts, mountLabel)); err != nil {
-		return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
-	}
-	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
-	// user namespace requires this to move a directory from lower to upper.
-	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
-	if err != nil {
-		return "", err
-	}
-	if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil {
-		return "", err
-	}
-	return mergedDir, nil
-}
-
-// Put unmounts the mount path created for the give id.
-func (d *Driver) Put(id string) error {
-	mountpoint := path.Join(d.dir(id), "merged")
-	if count := d.ctr.Decrement(mountpoint); count > 0 {
-		return nil
-	}
-	err := syscall.Unmount(mountpoint, 0)
-	if err != nil {
-		rootDir := path.Join(d.dir(id), "root")
-		if _, err := os.Stat(rootDir); err == nil {
-			// We weren't mounting a "merged" directory anyway
-			return nil
-		}
-		logrus.Debugf("Failed to unmount %s overlay: %v", id, err)
-	}
-	return err
-}
-
-// ApplyDiff applies the new layer on top of the root, if parent does not exist with will return an ErrApplyDiffFallback error.
-func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size int64, err error) {
-	dir := d.dir(id)
-
-	if parent == "" {
-		return 0, ErrApplyDiffFallback
-	}
-
-	parentRootDir := path.Join(d.dir(parent), "root")
-	if _, err := os.Stat(parentRootDir); err != nil {
-		return 0, ErrApplyDiffFallback
-	}
-
-	// We now know there is a parent, and it has a "root" directory containing
-	// the full root filesystem. We can just hardlink it and apply the
-	// layer. This relies on two things:
-	// 1) ApplyDiff is only run once on a clean (no writes to upper layer) container
-	// 2) ApplyDiff doesn't do any in-place writes to files (would break hardlinks)
-	// These are all currently true and are not expected to break
-
-	tmpRootDir, err := ioutil.TempDir(dir, "tmproot")
-	if err != nil {
-		return 0, err
-	}
-	defer func() {
-		if err != nil {
-			os.RemoveAll(tmpRootDir)
-		} else {
-			os.RemoveAll(path.Join(dir, "upper"))
-			os.RemoveAll(path.Join(dir, "work"))
-			os.RemoveAll(path.Join(dir, "merged"))
-			os.RemoveAll(path.Join(dir, "lower-id"))
-		}
-	}()
-
-	if err = copyDir(parentRootDir, tmpRootDir, copyHardlink); err != nil {
-		return 0, err
-	}
-
-	options := &archive.TarOptions{UIDMaps: d.uidMaps, GIDMaps: d.gidMaps}
-	if size, err = graphdriver.ApplyUncompressedLayer(tmpRootDir, diff, options); err != nil {
-		return 0, err
-	}
-
-	rootDir := path.Join(dir, "root")
-	if err := os.Rename(tmpRootDir, rootDir); err != nil {
-		return 0, err
-	}
-
-	return
-}
-
-// Exists checks to see if the id is already mounted.
-func (d *Driver) Exists(id string) bool {
-	_, err := os.Stat(d.dir(id))
-	return err == nil
-}

vendor/github.com/containers/storage/drivers/overlay/overlay_unsupported.go

@@ -1,3 +0,0 @@
-// +build !linux
-
-package overlay

vendor/github.com/containers/storage/drivers/overlay2/overlay.go

@@ -25,7 +25,7 @@ import (
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/parsers/kernel"
 
-	"github.com/opencontainers/runc/libcontainer/label"
+	"github.com/opencontainers/selinux/go-selinux/label"
 )
 
 var (
@@ -172,7 +172,7 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				return nil, err
 			}
 		default:
-			return nil, fmt.Errorf("overlay2: Unknown option %s\n", key)
+			return nil, fmt.Errorf("overlay2: Unknown option %s", key)
 		}
 	}
 	return o, nil

vendor/github.com/containers/storage/drivers/register/register_overlay.go

@@ -3,7 +3,6 @@
 package register
 
 import (
-	// register the overlay graphdriver
-	_ "github.com/containers/storage/drivers/overlay"
+	// register the overlay2 graphdriver
 	_ "github.com/containers/storage/drivers/overlay2"
 )

vendor/github.com/containers/storage/drivers/vfs/driver.go

@@ -9,7 +9,7 @@ import (
 	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/idtools"
 
-	"github.com/opencontainers/runc/libcontainer/label"
+	"github.com/opencontainers/selinux/go-selinux/label"
 )
 
 var (

vendor/github.com/containers/storage/drivers/zfs/zfs.go

@@ -19,7 +19,7 @@ import (
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
 	zfs "github.com/mistifyio/go-zfs"
-	"github.com/opencontainers/runc/libcontainer/label"
+	"github.com/opencontainers/selinux/go-selinux/label"
 )
 
 type zfsOptions struct {

vendor/github.com/containers/storage/pkg/ioutils/readers.go

@@ -4,8 +4,6 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"io"
-
-	"golang.org/x/net/context"
 )
 
 type readCloserWrapper struct {
@@ -83,72 +81,3 @@ func (r *OnEOFReader) runFunc() {
 		r.Fn = nil
 	}
 }
-
-// cancelReadCloser wraps an io.ReadCloser with a context for cancelling read
-// operations.
-type cancelReadCloser struct {
-	cancel func()
-	pR     *io.PipeReader // Stream to read from
-	pW     *io.PipeWriter
-}
-
-// NewCancelReadCloser creates a wrapper that closes the ReadCloser when the
-// context is cancelled. The returned io.ReadCloser must be closed when it is
-// no longer needed.
-func NewCancelReadCloser(ctx context.Context, in io.ReadCloser) io.ReadCloser {
-	pR, pW := io.Pipe()
-
-	// Create a context used to signal when the pipe is closed
-	doneCtx, cancel := context.WithCancel(context.Background())
-
-	p := &cancelReadCloser{
-		cancel: cancel,
-		pR:     pR,
-		pW:     pW,
-	}
-
-	go func() {
-		_, err := io.Copy(pW, in)
-		select {
-		case <-ctx.Done():
-			// If the context was closed, p.closeWithError
-			// was already called. Calling it again would
-			// change the error that Read returns.
-		default:
-			p.closeWithError(err)
-		}
-		in.Close()
-	}()
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				p.closeWithError(ctx.Err())
-			case <-doneCtx.Done():
-				return
-			}
-		}
-	}()
-
-	return p
-}
-
-// Read wraps the Read method of the pipe that provides data from the wrapped
-// ReadCloser.
-func (p *cancelReadCloser) Read(buf []byte) (n int, err error) {
-	return p.pR.Read(buf)
-}
-
-// closeWithError closes the wrapper and its underlying reader. It will
-// cause future calls to Read to return err.
-func (p *cancelReadCloser) closeWithError(err error) {
-	p.pW.CloseWithError(err)
-	p.cancel()
-}
-
-// Close closes the wrapper its underlying reader. It will cause
-// future calls to Read to return io.EOF.
-func (p *cancelReadCloser) Close() error {
-	p.closeWithError(io.EOF)
-	return nil
-}

vendor/github.com/containers/storage/storage/containers.go

@@ -151,6 +151,9 @@ func (r *containerStore) Load() error {
 
 func (r *containerStore) Save() error {
 	rpath := r.containerspath()
+	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
+		return err
+	}
 	jdata, err := json.Marshal(&r.containers)
 	if err != nil {
 		return err

vendor/github.com/containers/storage/storage/images.go

@@ -142,6 +142,9 @@ func (r *imageStore) Load() error {
 
 func (r *imageStore) Save() error {
 	rpath := r.imagespath()
+	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
+		return err
+	}
 	jdata, err := json.Marshal(&r.images)
 	if err != nil {
 		return err

vendor/github.com/containers/storage/storage/layers.go

@@ -252,11 +252,17 @@ func (r *layerStore) Load() error {
 
 func (r *layerStore) Save() error {
 	rpath := r.layerspath()
+	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
+		return err
+	}
 	jldata, err := json.Marshal(&r.layers)
 	if err != nil {
 		return err
 	}
 	mpath := r.mountspath()
+	if err := os.MkdirAll(filepath.Dir(mpath), 0700); err != nil {
+		return err
+	}
 	mounts := []layerMountPoint{}
 	for _, layer := range r.layers {
 		if layer.MountPoint != "" && layer.MountCount > 0 {
@@ -846,6 +852,9 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err
 	size, err = r.driver.ApplyDiff(layer.ID, layer.Parent, payload)
 	compressor.Close()
 	if err == nil {
+		if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil {
+			return -1, err
+		}
 		if err := ioutils.AtomicWriteFile(r.tspath(layer.ID), tsdata.Bytes(), 0600); err != nil {
 			return -1, err
 		}

vendor/github.com/containers/storage/storage/store.go

@@ -749,19 +749,25 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat
 		id = stringid.GenerateRandomID()
 	}
 
-	cimage, err := ristore.Get(image)
-	if err != nil {
-		return nil, err
+	imageTopLayer := ""
+	imageID := ""
+	if image != "" {
+		cimage, err := ristore.Get(image)
+		if err != nil {
+			return nil, err
+		}
+		if cimage == nil {
+			return nil, ErrImageUnknown
+		}
+		imageTopLayer = cimage.TopLayer
+		imageID = cimage.ID
 	}
-	if cimage == nil {
-		return nil, ErrImageUnknown
-	}
-	clayer, err := rlstore.Create(layer, cimage.TopLayer, nil, "", nil, true)
+	clayer, err := rlstore.Create(layer, imageTopLayer, nil, "", nil, true)
 	if err != nil {
 		return nil, err
 	}
 	layer = clayer.ID
-	container, err := rcstore.Create(id, names, cimage.ID, layer, metadata)
+	container, err := rcstore.Create(id, names, imageID, layer, metadata)
 	if err != nil || container == nil {
 		rlstore.Delete(layer)
 	}
@@ -2171,8 +2177,8 @@ func makeBigDataBaseName(key string) string {
 }
 
 func init() {
-	DefaultStoreOptions.RunRoot = "/var/run/containers"
-	DefaultStoreOptions.GraphRoot = "/var/lib/containers"
+	DefaultStoreOptions.RunRoot = "/var/run/containers/storage"
+	DefaultStoreOptions.GraphRoot = "/var/lib/containers/storage"
 	DefaultStoreOptions.GraphDriverName = os.Getenv("STORAGE_DRIVER")
 	DefaultStoreOptions.GraphDriverOptions = strings.Split(os.Getenv("STORAGE_OPTS"), ",")
 	if len(DefaultStoreOptions.GraphDriverOptions) == 1 && DefaultStoreOptions.GraphDriverOptions[0] == "" {

vendor/github.com/docker/distribution/context/doc.go

@@ -64,7 +64,7 @@
 // Note that this only affects the new context, the previous context, with the
 // version field, can be used independently. Put another way, the new logger,
 // added to the request context, is unique to that context and can have
-// request scoped varaibles.
+// request scoped variables.
 //
 // HTTP Requests
 //

vendor/github.com/docker/distribution/registry/api/v2/urls.go

@@ -1,10 +1,9 @@
 package v2
 
 import (
-	"net"
+	"fmt"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 
 	"github.com/docker/distribution/reference"
@@ -48,66 +47,42 @@ func NewURLBuilderFromString(root string, relative bool) (*URLBuilder, error) {
 // NewURLBuilderFromRequest uses information from an *http.Request to
 // construct the root url.
 func NewURLBuilderFromRequest(r *http.Request, relative bool) *URLBuilder {
-	var scheme string
-
-	forwardedProto := r.Header.Get("X-Forwarded-Proto")
-	// TODO: log the error
-	forwardedHeader, _, _ := parseForwardedHeader(r.Header.Get("Forwarded"))
-
-	switch {
-	case len(forwardedProto) > 0:
-		scheme = forwardedProto
-	case len(forwardedHeader["proto"]) > 0:
-		scheme = forwardedHeader["proto"]
-	case r.TLS != nil:
-		scheme = "https"
-	case len(r.URL.Scheme) > 0:
-		scheme = r.URL.Scheme
-	default:
+	var (
 		scheme = "http"
+		host   = r.Host
+	)
+
+	if r.TLS != nil {
+		scheme = "https"
+	} else if len(r.URL.Scheme) > 0 {
+		scheme = r.URL.Scheme
 	}
 
-	host := r.Host
-
-	if forwardedHost := r.Header.Get("X-Forwarded-Host"); len(forwardedHost) > 0 {
-		// According to the Apache mod_proxy docs, X-Forwarded-Host can be a
-		// comma-separated list of hosts, to which each proxy appends the
-		// requested host. We want to grab the first from this comma-separated
-		// list.
-		hosts := strings.SplitN(forwardedHost, ",", 2)
-		host = strings.TrimSpace(hosts[0])
-	} else if addr, exists := forwardedHeader["for"]; exists {
-		host = addr
-	} else if h, exists := forwardedHeader["host"]; exists {
-		host = h
-	}
-
-	portLessHost, port := host, ""
-	if !isIPv6Address(portLessHost) {
-		// with go 1.6, this would treat the last part of IPv6 address as a port
-		portLessHost, port, _ = net.SplitHostPort(host)
-	}
-	if forwardedPort := r.Header.Get("X-Forwarded-Port"); len(port) == 0 && len(forwardedPort) > 0 {
-		ports := strings.SplitN(forwardedPort, ",", 2)
-		forwardedPort = strings.TrimSpace(ports[0])
-		if _, err := strconv.ParseInt(forwardedPort, 10, 32); err == nil {
-			port = forwardedPort
+	// Handle fowarded headers
+	// Prefer "Forwarded" header as defined by rfc7239 if given
+	// see https://tools.ietf.org/html/rfc7239
+	if forwarded := r.Header.Get("Forwarded"); len(forwarded) > 0 {
+		forwardedHeader, _, err := parseForwardedHeader(forwarded)
+		if err == nil {
+			if fproto := forwardedHeader["proto"]; len(fproto) > 0 {
+				scheme = fproto
+			}
+			if fhost := forwardedHeader["host"]; len(fhost) > 0 {
+				host = fhost
+			}
 		}
-	}
-
-	if len(portLessHost) > 0 {
-		host = portLessHost
-	}
-	if len(port) > 0 {
-		// remove enclosing brackets of ipv6 address otherwise they will be duplicated
-		if len(host) > 1 && host[0] == '[' && host[len(host)-1] == ']' {
-			host = host[1 : len(host)-1]
+	} else {
+		if forwardedProto := r.Header.Get("X-Forwarded-Proto"); len(forwardedProto) > 0 {
+			scheme = forwardedProto
+		}
+		if forwardedHost := r.Header.Get("X-Forwarded-Host"); len(forwardedHost) > 0 {
+			// According to the Apache mod_proxy docs, X-Forwarded-Host can be a
+			// comma-separated list of hosts, to which each proxy appends the
+			// requested host. We want to grab the first from this comma-separated
+			// list.
+			hosts := strings.SplitN(forwardedHost, ",", 2)
+			host = strings.TrimSpace(hosts[0])
 		}
-		// JoinHostPort properly encloses ipv6 addresses in square brackets
-		host = net.JoinHostPort(host, port)
-	} else if isIPv6Address(host) && host[0] != '[' {
-		// ipv6 needs to be enclosed in square brackets in urls
-		host = "[" + host + "]"
 	}
 
 	basePath := routeDescriptorsMap[RouteNameBase].Path
@@ -175,6 +150,8 @@ func (ub *URLBuilder) BuildManifestURL(ref reference.Named) (string, error) {
 		tagOrDigest = v.Tag()
 	case reference.Digested:
 		tagOrDigest = v.Digest().String()
+	default:
+		return "", fmt.Errorf("reference must have a tag or digest")
 	}
 
 	manifestURL, err := route.URL("name", ref.Name(), "reference", tagOrDigest)
@@ -287,28 +264,3 @@ func appendValues(u string, values ...url.Values) string {
 
 	return appendValuesURL(up, values...).String()
 }
-
-// isIPv6Address returns true if given string is a valid IPv6 address. No port is allowed. The address may be
-// enclosed in square brackets.
-func isIPv6Address(host string) bool {
-	if len(host) > 1 && host[0] == '[' && host[len(host)-1] == ']' {
-		host = host[1 : len(host)-1]
-	}
-	// The IPv6 scoped addressing zone identifier starts after the last percent sign.
-	if i := strings.LastIndexByte(host, '%'); i > 0 {
-		host = host[:i]
-	}
-	ip := net.ParseIP(host)
-	if ip == nil {
-		return false
-	}
-	if ip.To16() == nil {
-		return false
-	}
-	if ip.To4() == nil {
-		return true
-	}
-	// dot can be present in ipv4-mapped address, it needs to come after a colon though
-	i := strings.IndexAny(host, ":.")
-	return i >= 0 && host[i] == ':'
-}

vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c

@@ -72,18 +72,23 @@ struct nlconfig_t {
 	char *namespaces;
 	size_t namespaces_len;
 	uint8_t is_setgroup;
+	uint8_t is_rootless;
+	char *oom_score_adj;
+	size_t oom_score_adj_len;
 };
 
 /*
  * List of netlink message types sent to us as part of bootstrapping the init.
  * These constants are defined in libcontainer/message_linux.go.
  */
-#define INIT_MSG		62000
+#define INIT_MSG			62000
 #define CLONE_FLAGS_ATTR	27281
 #define NS_PATHS_ATTR		27282
-#define UIDMAP_ATTR		27283
-#define GIDMAP_ATTR		27284
+#define UIDMAP_ATTR			27283
+#define GIDMAP_ATTR			27284
 #define SETGROUP_ATTR		27285
+#define OOM_SCORE_ADJ_ATTR	27286
+#define ROOTLESS_ATTR	    27287
 
 /*
  * Use the raw syscall for versions of glibc which don't include a function for
@@ -172,6 +177,7 @@ static void update_setgroups(int pid, enum policy_t setgroup)
 			policy = "deny";
 			break;
 		case SETGROUPS_DEFAULT:
+		default:
 			/* Nothing to do. */
 			return;
 	}
@@ -186,7 +192,7 @@ static void update_setgroups(int pid, enum policy_t setgroup)
 	}
 }
 
-static void update_uidmap(int pid, char *map, int map_len)
+static void update_uidmap(int pid, char *map, size_t map_len)
 {
 	if (map == NULL || map_len <= 0)
 		return;
@@ -195,7 +201,7 @@ static void update_uidmap(int pid, char *map, int map_len)
 		bail("failed to update /proc/%d/uid_map", pid);
 }
 
-static void update_gidmap(int pid, char *map, int map_len)
+static void update_gidmap(int pid, char *map, size_t map_len)
 {
 	if (map == NULL || map_len <= 0)
 		return;
@@ -204,6 +210,15 @@ static void update_gidmap(int pid, char *map, int map_len)
 		bail("failed to update /proc/%d/gid_map", pid);
 }
 
+static void update_oom_score_adj(char *data, size_t len)
+{
+	if (data == NULL || len <= 0)
+		return;
+
+	if (write_file(data, len, "/proc/self/oom_score_adj") < 0)
+		bail("failed to update /proc/self/oom_score_adj");
+}
+
 /* A dummy function that just jumps to the given jumpval. */
 static int child_func(void *arg) __attribute__ ((noinline));
 static int child_func(void *arg)
@@ -317,6 +332,13 @@ static void nl_parse(int fd, struct nlconfig_t *config)
 		case CLONE_FLAGS_ATTR:
 			config->cloneflags = readint32(current);
 			break;
+		case ROOTLESS_ATTR:
+			config->is_rootless = readint8(current);
+			break;
+		case OOM_SCORE_ADJ_ATTR:
+			config->oom_score_adj = current;
+			config->oom_score_adj_len = payload_len;
+			break;
 		case NS_PATHS_ATTR:
 			config->namespaces = current;
 			config->namespaces_len = payload_len;
@@ -425,14 +447,32 @@ void nsexec(void)
 	if (pipenum == -1)
 		return;
 
-	/* make the process non-dumpable */
-	if (prctl(PR_SET_DUMPABLE, 0, 0, 0, 0) != 0) {
-		bail("failed to set process as non-dumpable");
-	}
-
 	/* Parse all of the netlink configuration. */
 	nl_parse(pipenum, &config);
 
+	/* Set oom_score_adj. This has to be done before !dumpable because
+	 * /proc/self/oom_score_adj is not writeable unless you're an privileged
+	 * user (if !dumpable is set). All children inherit their parent's
+	 * oom_score_adj value on fork(2) so this will always be propagated
+	 * properly.
+	 */
+	update_oom_score_adj(config.oom_score_adj, config.oom_score_adj_len);
+
+	/*
+	 * Make the process non-dumpable, to avoid various race conditions that
+	 * could cause processes in namespaces we're joining to access host
+	 * resources (or potentially execute code).
+	 *
+	 * However, if the number of namespaces we are joining is 0, we are not
+	 * going to be switching to a different security context. Thus setting
+	 * ourselves to be non-dumpable only breaks things (like rootless
+	 * containers), which is the recommendation from the kernel folks.
+	 */
+	if (config.namespaces) {
+		if (prctl(PR_SET_DUMPABLE, 0, 0, 0, 0) < 0)
+			bail("failed to set process as non-dumpable");
+	}
+
 	/* Pipe so we can tell the child when we've finished setting up. */
 	if (socketpair(AF_LOCAL, SOCK_STREAM, 0, sync_child_pipe) < 0)
 		bail("failed to setup sync pipe between parent and child");
@@ -540,9 +580,21 @@ void nsexec(void)
 
 					exit(ret);
 				case SYNC_USERMAP_PLS:
-					/* Enable setgroups(2) if we've been asked to. */
+					/*
+					 * Enable setgroups(2) if we've been asked to. But we also
+					 * have to explicitly disable setgroups(2) if we're
+					 * creating a rootless container (this is required since
+					 * Linux 3.19).
+					 */
+					if (config.is_rootless && config.is_setgroup) {
+						kill(child, SIGKILL);
+						bail("cannot allow setgroup in an unprivileged user namespace setup");
+					}
+
 					if (config.is_setgroup)
 						update_setgroups(child, SETGROUPS_ALLOW);
+					if (config.is_rootless)
+						update_setgroups(child, SETGROUPS_DENY);
 
 					/* Set up mappings. */
 					update_uidmap(child, config.uidmap, config.uidmap_len);
@@ -681,6 +733,11 @@ void nsexec(void)
 				 * clone_parent rant). So signal our parent to hook us up.
 				 */
 
+				/* Switching is only necessary if we joined namespaces. */
+				if (config.namespaces) {
+					if (prctl(PR_SET_DUMPABLE, 1, 0, 0, 0) < 0)
+						bail("failed to set process as dumpable");
+				}
 				s = SYNC_USERMAP_PLS;
 				if (write(syncfd, &s, sizeof(s)) != sizeof(s))
 					bail("failed to sync with parent: write(SYNC_USERMAP_PLS)");
@@ -691,6 +748,11 @@ void nsexec(void)
 					bail("failed to sync with parent: read(SYNC_USERMAP_ACK)");
 				if (s != SYNC_USERMAP_ACK)
 					bail("failed to sync with parent: SYNC_USERMAP_ACK: got %u", s);
+				/* Switching is only necessary if we joined namespaces. */
+				if (config.namespaces) {
+					if (prctl(PR_SET_DUMPABLE, 0, 0, 0, 0) < 0)
+						bail("failed to set process as dumpable");
+				}
 			}
 
 			/*
@@ -774,8 +836,10 @@ void nsexec(void)
 			if (setgid(0) < 0)
 				bail("setgid failed");
 
-			if (setgroups(0, NULL) < 0)
-				bail("setgroups failed");
+			if (!config.is_rootless && config.is_setgroup) {
+				if (setgroups(0, NULL) < 0)
+					bail("setgroups failed");
+			}
 
 			s = SYNC_CHILD_READY;
 			if (write(syncfd, &s, sizeof(s)) != sizeof(s))

vendor/github.com/opencontainers/runc/libcontainer/user/user.go

@@ -199,18 +199,16 @@ type ExecUser struct {
 // files cannot be opened for any reason, the error is ignored and a nil
 // io.Reader is passed instead.
 func GetExecUserPath(userSpec string, defaults *ExecUser, passwdPath, groupPath string) (*ExecUser, error) {
-	passwd, err := os.Open(passwdPath)
-	if err != nil {
-		passwd = nil
-	} else {
-		defer passwd.Close()
+	var passwd, group io.Reader
+
+	if passwdFile, err := os.Open(passwdPath); err == nil {
+		passwd = passwdFile
+		defer passwdFile.Close()
 	}
 
-	group, err := os.Open(groupPath)
-	if err != nil {
-		group = nil
-	} else {
-		defer group.Close()
+	if groupFile, err := os.Open(groupPath); err == nil {
+		group = groupFile
+		defer groupFile.Close()
 	}
 
 	return GetExecUser(userSpec, defaults, passwd, group)
@@ -433,9 +431,11 @@ func GetAdditionalGroups(additionalGroups []string, group io.Reader) ([]int, err
 // that opens the groupPath given and gives it as an argument to
 // GetAdditionalGroups.
 func GetAdditionalGroupsPath(additionalGroups []string, groupPath string) ([]int, error) {
-	group, err := os.Open(groupPath)
-	if err == nil {
-		defer group.Close()
+	var group io.Reader
+
+	if groupFile, err := os.Open(groupPath); err == nil {
+		group = groupFile
+		defer groupFile.Close()
 	}
 	return GetAdditionalGroups(additionalGroups, group)
 }

vendor/github.com/opencontainers/runc/libcontainer/utils/utils_unix.go

@@ -4,6 +4,7 @@ package utils
 
 import (
 	"io/ioutil"
+	"os"
 	"strconv"
 	"syscall"
 )
@@ -31,3 +32,12 @@ func CloseExecFrom(minFd int) error {
 	}
 	return nil
 }
+
+// NewSockPair returns a new unix socket pair
+func NewSockPair(name string) (parent *os.File, child *os.File, err error) {
+	fds, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	return os.NewFile(uintptr(fds[1]), name+"-p"), os.NewFile(uintptr(fds[0]), name+"-c"), nil
+}

vendor/github.com/opencontainers/selinux/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/opencontainers/selinux/README.md

@@ -0,0 +1,7 @@
+# selinux
+
+[![GoDoc](https://godoc.org/github.com/opencontainers/selinux?status.svg)](https://godoc.org/github.com/opencontainers/selinux) [![Go Report Card](https://goreportcard.com/badge/github.com/opencontainers/selinux)](https://goreportcard.com/report/github.com/opencontainers/selinux) [![Build Status](https://travis-ci.org/opencontainers/selinux.svg?branch=master)](https://travis-ci.org/opencontainers/selinux)
+
+Common SELinux package used across the container ecosystem.
+
+Please see the [godoc](https://godoc.org/github.com/opencontainers/selinux) for more information.

vendor/github.com/opencontainers/selinux/go-selinux/label/label.go

@@ -52,7 +52,7 @@ func ReserveLabel(label string) error {
 	return nil
 }
 
-func UnreserveLabel(label string) error {
+func ReleaseLabel(label string) error {
 	return nil
 }
 

vendor/github.com/opencontainers/selinux/go-selinux/label/label_selinux.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/opencontainers/runc/libcontainer/selinux"
+	"github.com/opencontainers/selinux/go-selinux"
 )
 
 // Valid Label Options
@@ -25,10 +25,10 @@ var ErrIncompatibleLabel = fmt.Errorf("Bad SELinux option z and Z can not be use
 // the labels.  The labels returned will include a random MCS String, that is
 // guaranteed to be unique.
 func InitLabels(options []string) (string, string, error) {
-	if !selinux.SelinuxEnabled() {
+	if !selinux.GetEnabled() {
 		return "", "", nil
 	}
-	processLabel, mountLabel := selinux.GetLxcContexts()
+	processLabel, mountLabel := selinux.ContainerLabels()
 	if processLabel != "" {
 		pcon := selinux.NewContext(processLabel)
 		mcon := selinux.NewContext(mountLabel)
@@ -55,8 +55,8 @@ func InitLabels(options []string) (string, string, error) {
 	return processLabel, mountLabel, nil
 }
 
-func GetROMountLabel() string {
-	return selinux.GetROFileLabel()
+func ROMountLabel() string {
+	return selinux.ROFileLabel()
 }
 
 // DEPRECATED: The GenLabels function is only to be used during the transition to the official API.
@@ -88,33 +88,33 @@ func SetProcessLabel(processLabel string) error {
 	if processLabel == "" {
 		return nil
 	}
-	return selinux.Setexeccon(processLabel)
+	return selinux.SetExecLabel(processLabel)
 }
 
-// GetProcessLabel returns the process label that the kernel will assign
+// ProcessLabel returns the process label that the kernel will assign
 // to the next program executed by the current process.  If "" is returned
 // this indicates that the default labeling will happen for the process.
-func GetProcessLabel() (string, error) {
-	return selinux.Getexeccon()
+func ProcessLabel() (string, error) {
+	return selinux.ExecLabel()
 }
 
 // GetFileLabel returns the label for specified path
-func GetFileLabel(path string) (string, error) {
-	return selinux.Getfilecon(path)
+func FileLabel(path string) (string, error) {
+	return selinux.FileLabel(path)
 }
 
 // SetFileLabel modifies the "path" label to the specified file label
 func SetFileLabel(path string, fileLabel string) error {
-	if selinux.SelinuxEnabled() && fileLabel != "" {
-		return selinux.Setfilecon(path, fileLabel)
+	if selinux.GetEnabled() && fileLabel != "" {
+		return selinux.SetFileLabel(path, fileLabel)
 	}
 	return nil
 }
 
 // SetFileCreateLabel tells the kernel the label for all files to be created
 func SetFileCreateLabel(fileLabel string) error {
-	if selinux.SelinuxEnabled() {
-		return selinux.Setfscreatecon(fileLabel)
+	if selinux.GetEnabled() {
+		return selinux.SetFSCreateLabel(fileLabel)
 	}
 	return nil
 }
@@ -123,7 +123,7 @@ func SetFileCreateLabel(fileLabel string) error {
 // It changes the MCS label to s0 if shared is true.
 // This will allow all containers to share the content.
 func Relabel(path string, fileLabel string, shared bool) error {
-	if !selinux.SelinuxEnabled() {
+	if !selinux.GetEnabled() {
 		return nil
 	}
 
@@ -147,14 +147,14 @@ func Relabel(path string, fileLabel string, shared bool) error {
 	return nil
 }
 
-// GetPidLabel will return the label of the process running with the specified pid
-func GetPidLabel(pid int) (string, error) {
-	return selinux.Getpidcon(pid)
+// PidLabel will return the label of the process running with the specified pid
+func PidLabel(pid int) (string, error) {
+	return selinux.PidLabel(pid)
 }
 
 // Init initialises the labeling system
 func Init() {
-	selinux.SelinuxEnabled()
+	selinux.GetEnabled()
 }
 
 // ReserveLabel will record the fact that the MCS label has already been used.
@@ -165,11 +165,11 @@ func ReserveLabel(label string) error {
 	return nil
 }
 
-// UnreserveLabel will remove the reservation of the MCS label.
+// ReleaseLabel will remove the reservation of the MCS label.
 // This will allow InitLabels to use the MCS label in a newly created
 // containers
-func UnreserveLabel(label string) error {
-	selinux.FreeLxcContexts(label)
+func ReleaseLabel(label string) error {
+	selinux.ReleaseLabel(label)
 	return nil
 }
 

vendor/github.com/opencontainers/selinux/go-selinux/selinux.go

@@ -15,13 +15,14 @@ import (
 	"strings"
 	"sync"
 	"syscall"
-
-	"github.com/opencontainers/runc/libcontainer/system"
 )
 
 const (
-	Enforcing        = 1
-	Permissive       = 0
+	// Enforcing constant indicate SELinux is in enforcing mode
+	Enforcing = 1
+	// Permissive constant to indicate SELinux is in permissive mode
+	Permissive = 0
+	// Disabled constant to indicate SELinux is disabled
 	Disabled         = -1
 	selinuxDir       = "/etc/selinux/"
 	selinuxConfig    = selinuxDir + "config"
@@ -48,7 +49,8 @@ var (
 	}
 )
 
-type SELinuxContext map[string]string
+// Context is a representation of the SELinux label broken into 4 parts
+type Context map[string]string
 
 func (s *selinuxState) setEnable(enabled bool) bool {
 	s.Lock()
@@ -69,7 +71,7 @@ func (s *selinuxState) getEnabled() bool {
 
 	enabled = false
 	if fs := getSelinuxMountPoint(); fs != "" {
-		if con, _ := Getcon(); con != "kernel" {
+		if con, _ := CurrentLabel(); con != "kernel" {
 			enabled = true
 		}
 	}
@@ -143,8 +145,8 @@ func getSelinuxMountPoint() string {
 	return state.getSELinuxfs()
 }
 
-// SelinuxEnabled returns whether selinux is currently enabled.
-func SelinuxEnabled() bool {
+// GetEnabled returns whether selinux is currently enabled.
+func GetEnabled() bool {
 	return state.getEnabled()
 }
 
@@ -206,43 +208,55 @@ func readCon(name string) (string, error) {
 	return val, err
 }
 
-// Setfilecon sets the SELinux label for this path or returns an error.
-func Setfilecon(path string, scon string) error {
-	return system.Lsetxattr(path, xattrNameSelinux, []byte(scon), 0)
+// SetFileLabel sets the SELinux label for this path or returns an error.
+func SetFileLabel(path string, label string) error {
+	return lsetxattr(path, xattrNameSelinux, []byte(label), 0)
 }
 
-// Getfilecon returns the SELinux label for this path or returns an error.
-func Getfilecon(path string) (string, error) {
-	con, err := system.Lgetxattr(path, xattrNameSelinux)
+// Filecon returns the SELinux label for this path or returns an error.
+func FileLabel(path string) (string, error) {
+	label, err := lgetxattr(path, xattrNameSelinux)
 	if err != nil {
 		return "", err
 	}
 	// Trim the NUL byte at the end of the byte buffer, if present.
-	if len(con) > 0 && con[len(con)-1] == '\x00' {
-		con = con[:len(con)-1]
+	if len(label) > 0 && label[len(label)-1] == '\x00' {
+		label = label[:len(label)-1]
 	}
-	return string(con), nil
+	return string(label), nil
 }
 
-func Setfscreatecon(scon string) error {
-	return writeCon(fmt.Sprintf("/proc/self/task/%d/attr/fscreate", syscall.Gettid()), scon)
+/*
+SetFSCreateLabel tells kernel the label to create all file system objects
+created by this task. Setting label="" to return to default.
+*/
+func SetFSCreateLabel(label string) error {
+	return writeCon(fmt.Sprintf("/proc/self/task/%d/attr/fscreate", syscall.Gettid()), label)
 }
 
-func Getfscreatecon() (string, error) {
+/*
+FSCreateLabel returns the default label the kernel which the kernel is using
+for file system objects created by this task. "" indicates default.
+*/
+func FSCreateLabel() (string, error) {
 	return readCon(fmt.Sprintf("/proc/self/task/%d/attr/fscreate", syscall.Gettid()))
 }
 
-// Getcon returns the SELinux label of the current process thread, or an error.
-func Getcon() (string, error) {
+// CurrentLabel returns the SELinux label of the current process thread, or an error.
+func CurrentLabel() (string, error) {
 	return readCon(fmt.Sprintf("/proc/self/task/%d/attr/current", syscall.Gettid()))
 }
 
-// Getpidcon returns the SELinux label of the given pid, or an error.
-func Getpidcon(pid int) (string, error) {
+// PidLabel returns the SELinux label of the given pid, or an error.
+func PidLabel(pid int) (string, error) {
 	return readCon(fmt.Sprintf("/proc/%d/attr/current", pid))
 }
 
-func Getexeccon() (string, error) {
+/*
+ExecLabel returns the SELinux label that the kernel will use for any programs
+that are executed by the current process thread, or an error.
+*/
+func ExecLabel() (string, error) {
 	return readCon(fmt.Sprintf("/proc/self/task/%d/attr/exec", syscall.Gettid()))
 }
 
@@ -261,19 +275,25 @@ func writeCon(name string, val string) error {
 	return err
 }
 
-func Setexeccon(scon string) error {
-	return writeCon(fmt.Sprintf("/proc/self/task/%d/attr/exec", syscall.Gettid()), scon)
+/*
+SetExecLabel sets the SELinux label that the kernel will use for any programs
+that are executed by the current process thread, or an error.
+*/
+func SetExecLabel(label string) error {
+	return writeCon(fmt.Sprintf("/proc/self/task/%d/attr/exec", syscall.Gettid()), label)
 }
 
-func (c SELinuxContext) Get() string {
+// Get returns the Context as a string
+func (c Context) Get() string {
 	return fmt.Sprintf("%s:%s:%s:%s", c["user"], c["role"], c["type"], c["level"])
 }
 
-func NewContext(scon string) SELinuxContext {
-	c := make(SELinuxContext)
+// NewContext creates a new Context struct from the specified label
+func NewContext(label string) Context {
+	c := make(Context)
 
-	if len(scon) != 0 {
-		con := strings.SplitN(scon, ":", 4)
+	if len(label) != 0 {
+		con := strings.SplitN(label, ":", 4)
 		c["user"] = con[0]
 		c["role"] = con[1]
 		c["type"] = con[2]
@@ -282,9 +302,10 @@ func NewContext(scon string) SELinuxContext {
 	return c
 }
 
-func ReserveLabel(scon string) {
-	if len(scon) != 0 {
-		con := strings.SplitN(scon, ":", 4)
+// ReserveLabel reserves the MLS/MCS level component of the specified label
+func ReserveLabel(label string) {
+	if len(label) != 0 {
+		con := strings.SplitN(label, ":", 4)
 		mcsAdd(con[3])
 	}
 }
@@ -293,7 +314,8 @@ func selinuxEnforcePath() string {
 	return fmt.Sprintf("%s/enforce", selinuxPath)
 }
 
-func SelinuxGetEnforce() int {
+// EnforceMode returns the current SELinux mode Enforcing, Permissive, Disabled
+func EnforceMode() int {
 	var enforce int
 
 	enforceS, err := readCon(selinuxEnforcePath())
@@ -308,11 +330,20 @@ func SelinuxGetEnforce() int {
 	return enforce
 }
 
-func SelinuxSetEnforce(mode int) error {
+/*
+SetEnforce sets the current SELinux mode Enforcing, Permissive.
+Disabled is not valid, since this needs to be set at boot time.
+*/
+func SetEnforceMode(mode int) error {
 	return writeCon(selinuxEnforcePath(), fmt.Sprintf("%d", mode))
 }
 
-func SelinuxGetEnforceMode() int {
+/*
+DefaultEnforceMode returns the systems default SELinux mode Enforcing,
+Permissive or Disabled. Note this is is just the default at boot time.
+EnforceMode tells you the systems current mode.
+*/
+func DefaultEnforceMode() int {
 	switch readConfig(selinuxTag) {
 	case "enforcing":
 		return Enforcing
@@ -338,7 +369,7 @@ func mcsDelete(mcs string) {
 	state.mcsList[mcs] = false
 }
 
-func IntToMcs(id int, catRange uint32) string {
+func intToMcs(id int, catRange uint32) string {
 	var (
 		SETSIZE = int(catRange)
 		TIER    = SETSIZE
@@ -386,26 +417,35 @@ func uniqMcs(catRange uint32) string {
 	return mcs
 }
 
-func FreeLxcContexts(scon string) {
-	if len(scon) != 0 {
-		con := strings.SplitN(scon, ":", 4)
+/*
+ReleaseLabel will unreserve the MLS/MCS Level field of the specified label.
+Allowing it to be used by another process.
+*/
+func ReleaseLabel(label string) {
+	if len(label) != 0 {
+		con := strings.SplitN(label, ":", 4)
 		mcsDelete(con[3])
 	}
 }
 
 var roFileLabel string
 
-func GetROFileLabel() (fileLabel string) {
+// ROFileLabel returns the specified SELinux readonly file label
+func ROFileLabel() (fileLabel string) {
 	return roFileLabel
 }
 
-func GetLxcContexts() (processLabel string, fileLabel string) {
+/*
+ContainerLabels returns an allocated processLabel and fileLabel to be used for
+container labeling by the calling process.
+*/
+func ContainerLabels() (processLabel string, fileLabel string) {
 	var (
 		val, key string
 		bufin    *bufio.Reader
 	)
 
-	if !SelinuxEnabled() {
+	if !GetEnabled() {
 		return "", ""
 	}
 	lxcPath := fmt.Sprintf("%s/contexts/lxc_contexts", getSELinuxPolicyRoot())
@@ -457,7 +497,6 @@ func GetLxcContexts() (processLabel string, fileLabel string) {
 		roFileLabel = fileLabel
 	}
 exit:
-	//	mcs := IntToMcs(os.Getpid(), 1024)
 	mcs := uniqMcs(1024)
 	scon := NewContext(processLabel)
 	scon["level"] = mcs
@@ -468,10 +507,15 @@ exit:
 	return processLabel, fileLabel
 }
 
+// SecurityCheckContext validates that the SELinux label is understood by the kernel
 func SecurityCheckContext(val string) error {
 	return writeCon(fmt.Sprintf("%s.context", selinuxPath), val)
 }
 
+/*
+CopyLevel returns a label with the MLS/MCS level from src label replaces on
+the dest label.
+*/
 func CopyLevel(src, dest string) (string, error) {
 	if src == "" {
 		return "", nil
@@ -502,25 +546,25 @@ func badPrefix(fpath string) error {
 	return nil
 }
 
-// Chcon changes the fpath file object to the SELinux label scon.
+// Chcon changes the fpath file object to the SELinux label label.
 // If the fpath is a directory and recurse is true Chcon will walk the
 // directory tree setting the label
-func Chcon(fpath string, scon string, recurse bool) error {
-	if scon == "" {
+func Chcon(fpath string, label string, recurse bool) error {
+	if label == "" {
 		return nil
 	}
 	if err := badPrefix(fpath); err != nil {
 		return err
 	}
 	callback := func(p string, info os.FileInfo, err error) error {
-		return Setfilecon(p, scon)
+		return SetFileLabel(p, label)
 	}
 
 	if recurse {
 		return filepath.Walk(fpath, callback)
 	}
 
-	return Setfilecon(fpath, scon)
+	return SetFileLabel(fpath, label)
 }
 
 // DupSecOpt takes an SELinux process label and returns security options that

vendor/github.com/opencontainers/selinux/go-selinux/xattrs.go

@@ -0,0 +1,78 @@
+// +build linux
+
+package selinux
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+var _zero uintptr
+
+// Returns a []byte slice if the xattr is set and nil otherwise
+// Requires path and its attribute as arguments
+func lgetxattr(path string, attr string) ([]byte, error) {
+	var sz int
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return nil, err
+	}
+	attrBytes, err := syscall.BytePtrFromString(attr)
+	if err != nil {
+		return nil, err
+	}
+
+	// Start with a 128 length byte array
+	sz = 128
+	dest := make([]byte, sz)
+	destBytes := unsafe.Pointer(&dest[0])
+	_sz, _, errno := syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(pathBytes)), uintptr(unsafe.Pointer(attrBytes)), uintptr(destBytes), uintptr(len(dest)), 0, 0)
+
+	switch {
+	case errno == syscall.ENODATA:
+		return nil, errno
+	case errno == syscall.ENOTSUP:
+		return nil, errno
+	case errno == syscall.ERANGE:
+		// 128 byte array might just not be good enough,
+		// A dummy buffer is used ``uintptr(0)`` to get real size
+		// of the xattrs on disk
+		_sz, _, errno = syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(pathBytes)), uintptr(unsafe.Pointer(attrBytes)), uintptr(unsafe.Pointer(nil)), uintptr(0), 0, 0)
+		sz = int(_sz)
+		if sz < 0 {
+			return nil, errno
+		}
+		dest = make([]byte, sz)
+		destBytes := unsafe.Pointer(&dest[0])
+		_sz, _, errno = syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(pathBytes)), uintptr(unsafe.Pointer(attrBytes)), uintptr(destBytes), uintptr(len(dest)), 0, 0)
+		if errno != 0 {
+			return nil, errno
+		}
+	case errno != 0:
+		return nil, errno
+	}
+	sz = int(_sz)
+	return dest[:sz], nil
+}
+
+func lsetxattr(path string, attr string, data []byte, flags int) error {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return err
+	}
+	attrBytes, err := syscall.BytePtrFromString(attr)
+	if err != nil {
+		return err
+	}
+	var dataBytes unsafe.Pointer
+	if len(data) > 0 {
+		dataBytes = unsafe.Pointer(&data[0])
+	} else {
+		dataBytes = unsafe.Pointer(&_zero)
+	}
+	_, _, errno := syscall.Syscall6(syscall.SYS_LSETXATTR, uintptr(unsafe.Pointer(pathBytes)), uintptr(unsafe.Pointer(attrBytes)), uintptr(dataBytes), uintptr(len(data)), uintptr(flags), 0)
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}

vendor/github.com/xeipuuv/gojsonschema/validation.go

@@ -568,7 +568,7 @@ func (v *subSchema) validateObject(currentSubSchema *subSchema, value map[string
 							result.addError(
 								new(AdditionalPropertyNotAllowedError),
 								context,
-								value,
+								value[pk],
 								ErrorDetails{"property": pk},
 							)
 						}
@@ -579,7 +579,7 @@ func (v *subSchema) validateObject(currentSubSchema *subSchema, value map[string
 							result.addError(
 								new(AdditionalPropertyNotAllowedError),
 								context,
-								value,
+								value[pk],
 								ErrorDetails{"property": pk},
 							)
 						}
@@ -631,7 +631,7 @@ func (v *subSchema) validateObject(currentSubSchema *subSchema, value map[string
 				result.addError(
 					new(InvalidPropertyPatternError),
 					context,
-					value,
+					value[pk],
 					ErrorDetails{
 						"property": pk,
 						"pattern":  currentSubSchema.PatternPropertiesString(),

vendor/golang.org/x/net/http2/frame.go

@@ -122,7 +122,7 @@ var flagName = map[FrameType]map[Flags]string{
 // a frameParser parses a frame given its FrameHeader and payload
 // bytes. The length of payload will always equal fh.Length (which
 // might be 0).
-type frameParser func(fh FrameHeader, payload []byte) (Frame, error)
+type frameParser func(fc *frameCache, fh FrameHeader, payload []byte) (Frame, error)
 
 var frameParsers = map[FrameType]frameParser{
 	FrameData:         parseDataFrame,
@@ -323,6 +323,8 @@ type Framer struct {
 	debugFramerBuf    *bytes.Buffer
 	debugReadLoggerf  func(string, ...interface{})
 	debugWriteLoggerf func(string, ...interface{})
+
+	frameCache *frameCache // nil if frames aren't reused (default)
 }
 
 func (fr *Framer) maxHeaderListSize() uint32 {
@@ -398,6 +400,27 @@ const (
 	maxFrameSize    = 1<<24 - 1
 )
 
+// SetReuseFrames allows the Framer to reuse Frames.
+// If called on a Framer, Frames returned by calls to ReadFrame are only
+// valid until the next call to ReadFrame.
+func (fr *Framer) SetReuseFrames() {
+	if fr.frameCache != nil {
+		return
+	}
+	fr.frameCache = &frameCache{}
+}
+
+type frameCache struct {
+	dataFrame DataFrame
+}
+
+func (fc *frameCache) getDataFrame() *DataFrame {
+	if fc == nil {
+		return &DataFrame{}
+	}
+	return &fc.dataFrame
+}
+
 // NewFramer returns a Framer that writes frames to w and reads them from r.
 func NewFramer(w io.Writer, r io.Reader) *Framer {
 	fr := &Framer{
@@ -477,7 +500,7 @@ func (fr *Framer) ReadFrame() (Frame, error) {
 	if _, err := io.ReadFull(fr.r, payload); err != nil {
 		return nil, err
 	}
-	f, err := typeFrameParser(fh.Type)(fh, payload)
+	f, err := typeFrameParser(fh.Type)(fr.frameCache, fh, payload)
 	if err != nil {
 		if ce, ok := err.(connError); ok {
 			return nil, fr.connError(ce.Code, ce.Reason)
@@ -565,7 +588,7 @@ func (f *DataFrame) Data() []byte {
 	return f.data
 }
 
-func parseDataFrame(fh FrameHeader, payload []byte) (Frame, error) {
+func parseDataFrame(fc *frameCache, fh FrameHeader, payload []byte) (Frame, error) {
 	if fh.StreamID == 0 {
 		// DATA frames MUST be associated with a stream. If a
 		// DATA frame is received whose stream identifier
@@ -574,9 +597,9 @@ func parseDataFrame(fh FrameHeader, payload []byte) (Frame, error) {
 		// PROTOCOL_ERROR.
 		return nil, connError{ErrCodeProtocol, "DATA frame with stream ID 0"}
 	}
-	f := &DataFrame{
-		FrameHeader: fh,
-	}
+	f := fc.getDataFrame()
+	f.FrameHeader = fh
+
 	var padSize byte
 	if fh.Flags.Has(FlagDataPadded) {
 		var err error
@@ -672,7 +695,7 @@ type SettingsFrame struct {
 	p []byte
 }
 
-func parseSettingsFrame(fh FrameHeader, p []byte) (Frame, error) {
+func parseSettingsFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) {
 	if fh.Flags.Has(FlagSettingsAck) && fh.Length > 0 {
 		// When this (ACK 0x1) bit is set, the payload of the
 		// SETTINGS frame MUST be empty. Receipt of a
@@ -774,7 +797,7 @@ type PingFrame struct {
 
 func (f *PingFrame) IsAck() bool { return f.Flags.Has(FlagPingAck) }
 
-func parsePingFrame(fh FrameHeader, payload []byte) (Frame, error) {
+func parsePingFrame(_ *frameCache, fh FrameHeader, payload []byte) (Frame, error) {
 	if len(payload) != 8 {
 		return nil, ConnectionError(ErrCodeFrameSize)
 	}
@@ -814,7 +837,7 @@ func (f *GoAwayFrame) DebugData() []byte {
 	return f.debugData
 }
 
-func parseGoAwayFrame(fh FrameHeader, p []byte) (Frame, error) {
+func parseGoAwayFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) {
 	if fh.StreamID != 0 {
 		return nil, ConnectionError(ErrCodeProtocol)
 	}
@@ -854,7 +877,7 @@ func (f *UnknownFrame) Payload() []byte {
 	return f.p
 }
 
-func parseUnknownFrame(fh FrameHeader, p []byte) (Frame, error) {
+func parseUnknownFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) {
 	return &UnknownFrame{fh, p}, nil
 }
 
@@ -865,7 +888,7 @@ type WindowUpdateFrame struct {
 	Increment uint32 // never read with high bit set
 }
 
-func parseWindowUpdateFrame(fh FrameHeader, p []byte) (Frame, error) {
+func parseWindowUpdateFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) {
 	if len(p) != 4 {
 		return nil, ConnectionError(ErrCodeFrameSize)
 	}
@@ -930,7 +953,7 @@ func (f *HeadersFrame) HasPriority() bool {
 	return f.FrameHeader.Flags.Has(FlagHeadersPriority)
 }
 
-func parseHeadersFrame(fh FrameHeader, p []byte) (_ Frame, err error) {
+func parseHeadersFrame(_ *frameCache, fh FrameHeader, p []byte) (_ Frame, err error) {
 	hf := &HeadersFrame{
 		FrameHeader: fh,
 	}
@@ -1067,7 +1090,7 @@ func (p PriorityParam) IsZero() bool {
 	return p == PriorityParam{}
 }
 
-func parsePriorityFrame(fh FrameHeader, payload []byte) (Frame, error) {
+func parsePriorityFrame(_ *frameCache, fh FrameHeader, payload []byte) (Frame, error) {
 	if fh.StreamID == 0 {
 		return nil, connError{ErrCodeProtocol, "PRIORITY frame with stream ID 0"}
 	}
@@ -1114,7 +1137,7 @@ type RSTStreamFrame struct {
 	ErrCode ErrCode
 }
 
-func parseRSTStreamFrame(fh FrameHeader, p []byte) (Frame, error) {
+func parseRSTStreamFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) {
 	if len(p) != 4 {
 		return nil, ConnectionError(ErrCodeFrameSize)
 	}
@@ -1144,7 +1167,7 @@ type ContinuationFrame struct {
 	headerFragBuf []byte
 }
 
-func parseContinuationFrame(fh FrameHeader, p []byte) (Frame, error) {
+func parseContinuationFrame(_ *frameCache, fh FrameHeader, p []byte) (Frame, error) {
 	if fh.StreamID == 0 {
 		return nil, connError{ErrCodeProtocol, "CONTINUATION frame with stream ID 0"}
 	}
@@ -1194,7 +1217,7 @@ func (f *PushPromiseFrame) HeadersEnded() bool {
 	return f.FrameHeader.Flags.Has(FlagPushPromiseEndHeaders)
 }
 
-func parsePushPromise(fh FrameHeader, p []byte) (_ Frame, err error) {
+func parsePushPromise(_ *frameCache, fh FrameHeader, p []byte) (_ Frame, err error) {
 	pp := &PushPromiseFrame{
 		FrameHeader: fh,
 	}

vendor/golang.org/x/net/http2/go18.go

@@ -12,7 +12,11 @@ import (
 	"net/http"
 )
 
-func cloneTLSConfig(c *tls.Config) *tls.Config { return c.Clone() }
+func cloneTLSConfig(c *tls.Config) *tls.Config {
+	c2 := c.Clone()
+	c2.GetClientCertificate = c.GetClientCertificate // golang.org/issue/19264
+	return c2
+}
 
 var _ http.Pusher = (*responseWriter)(nil)
 

vendor/golang.org/x/net/http2/hpack/encode.go

@@ -39,6 +39,7 @@ func NewEncoder(w io.Writer) *Encoder {
 		tableSizeUpdate: false,
 		w:               w,
 	}
+	e.dynTab.table.init()
 	e.dynTab.setMaxSize(initialHeaderTableSize)
 	return e
 }
@@ -88,24 +89,17 @@ func (e *Encoder) WriteField(f HeaderField) error {
 // only name matches, i points to that index and nameValueMatch
 // becomes false.
 func (e *Encoder) searchTable(f HeaderField) (i uint64, nameValueMatch bool) {
-	for idx, hf := range staticTable {
-		if hf.Name != f.Name {
-			continue
-		}
-		if i == 0 {
-			i = uint64(idx + 1)
-		}
-		if f.Sensitive || hf.Value != f.Value {
-			continue
-		}
-		return uint64(idx + 1), true
+	i, nameValueMatch = staticTable.search(f)
+	if nameValueMatch {
+		return i, true
 	}
 
-	j, nameValueMatch := e.dynTab.search(f)
+	j, nameValueMatch := e.dynTab.table.search(f)
 	if nameValueMatch || (i == 0 && j != 0) {
-		i = j + uint64(len(staticTable))
+		return j + uint64(staticTable.len()), nameValueMatch
 	}
-	return
+
+	return i, false
 }
 
 // SetMaxDynamicTableSize changes the dynamic header table size to v.

vendor/golang.org/x/net/http2/hpack/hpack.go

@@ -102,6 +102,7 @@ func NewDecoder(maxDynamicTableSize uint32, emitFunc func(f HeaderField)) *Decod
 		emit:        emitFunc,
 		emitEnabled: true,
 	}
+	d.dynTab.table.init()
 	d.dynTab.allowedMaxSize = maxDynamicTableSize
 	d.dynTab.setMaxSize(maxDynamicTableSize)
 	return d
@@ -154,12 +155,9 @@ func (d *Decoder) SetAllowedMaxDynamicTableSize(v uint32) {
 }
 
 type dynamicTable struct {
-	// ents is the FIFO described at
 	// http://http2.github.io/http2-spec/compression.html#rfc.section.2.3.2
-	// The newest (low index) is append at the end, and items are
-	// evicted from the front.
-	ents           []HeaderField
-	size           uint32
+	table          headerFieldTable
+	size           uint32 // in bytes
 	maxSize        uint32 // current maxSize
 	allowedMaxSize uint32 // maxSize may go up to this, inclusive
 }
@@ -169,77 +167,45 @@ func (dt *dynamicTable) setMaxSize(v uint32) {
 	dt.evict()
 }
 
-// TODO: change dynamicTable to be a struct with a slice and a size int field,
-// per http://http2.github.io/http2-spec/compression.html#rfc.section.4.1:
-//
-//
-// Then make add increment the size. maybe the max size should move from Decoder to
-// dynamicTable and add should return an ok bool if there was enough space.
-//
-// Later we'll need a remove operation on dynamicTable.
-
 func (dt *dynamicTable) add(f HeaderField) {
-	dt.ents = append(dt.ents, f)
+	dt.table.addEntry(f)
 	dt.size += f.Size()
 	dt.evict()
 }
 
-// If we're too big, evict old stuff (front of the slice)
+// If we're too big, evict old stuff.
 func (dt *dynamicTable) evict() {
-	base := dt.ents // keep base pointer of slice
-	for dt.size > dt.maxSize {
-		dt.size -= dt.ents[0].Size()
-		dt.ents = dt.ents[1:]
+	var n int
+	for dt.size > dt.maxSize && n < dt.table.len() {
+		dt.size -= dt.table.ents[n].Size()
+		n++
 	}
-
-	// Shift slice contents down if we evicted things.
-	if len(dt.ents) != len(base) {
-		copy(base, dt.ents)
-		dt.ents = base[:len(dt.ents)]
-	}
-}
-
-// Search searches f in the table. The return value i is 0 if there is
-// no name match. If there is name match or name/value match, i is the
-// index of that entry (1-based). If both name and value match,
-// nameValueMatch becomes true.
-func (dt *dynamicTable) search(f HeaderField) (i uint64, nameValueMatch bool) {
-	l := len(dt.ents)
-	for j := l - 1; j >= 0; j-- {
-		ent := dt.ents[j]
-		if ent.Name != f.Name {
-			continue
-		}
-		if i == 0 {
-			i = uint64(l - j)
-		}
-		if f.Sensitive {
-			continue
-		}
-		if ent.Value != f.Value {
-			continue
-		}
-		return uint64(l - j), true
-	}
-	return i, false
+	dt.table.evictOldest(n)
 }
 
 func (d *Decoder) maxTableIndex() int {
-	return len(d.dynTab.ents) + len(staticTable)
+	// This should never overflow. RFC 7540 Section 6.5.2 limits the size of
+	// the dynamic table to 2^32 bytes, where each entry will occupy more than
+	// one byte. Further, the staticTable has a fixed, small length.
+	return d.dynTab.table.len() + staticTable.len()
 }
 
 func (d *Decoder) at(i uint64) (hf HeaderField, ok bool) {
-	if i < 1 {
+	// See Section 2.3.3.
+	if i == 0 {
 		return
 	}
+	if i <= uint64(staticTable.len()) {
+		return staticTable.ents[i-1], true
+	}
 	if i > uint64(d.maxTableIndex()) {
 		return
 	}
-	if i <= uint64(len(staticTable)) {
-		return staticTable[i-1], true
-	}
-	dents := d.dynTab.ents
-	return dents[len(dents)-(int(i)-len(staticTable))], true
+	// In the dynamic table, newer entries have lower indices.
+	// However, dt.ents[0] is the oldest entry. Hence, dt.ents is
+	// the reversed dynamic table.
+	dt := d.dynTab.table
+	return dt.ents[dt.len()-(int(i)-staticTable.len())], true
 }
 
 // Decode decodes an entire block.

vendor/golang.org/x/net/http2/hpack/tables.go

@@ -4,73 +4,199 @@
 
 package hpack
 
+import (
+	"fmt"
+)
+
+// headerFieldTable implements a list of HeaderFields.
+// This is used to implement the static and dynamic tables.
+type headerFieldTable struct {
+	// For static tables, entries are never evicted.
+	//
+	// For dynamic tables, entries are evicted from ents[0] and added to the end.
+	// Each entry has a unique id that starts at one and increments for each
+	// entry that is added. This unique id is stable across evictions, meaning
+	// it can be used as a pointer to a specific entry. As in hpack, unique ids
+	// are 1-based. The unique id for ents[k] is k + evictCount + 1.
+	//
+	// Zero is not a valid unique id.
+	//
+	// evictCount should not overflow in any remotely practical situation. In
+	// practice, we will have one dynamic table per HTTP/2 connection. If we
+	// assume a very powerful server that handles 1M QPS per connection and each
+	// request adds (then evicts) 100 entries from the table, it would still take
+	// 2M years for evictCount to overflow.
+	ents       []HeaderField
+	evictCount uint64
+
+	// byName maps a HeaderField name to the unique id of the newest entry with
+	// the same name. See above for a definition of "unique id".
+	byName map[string]uint64
+
+	// byNameValue maps a HeaderField name/value pair to the unique id of the newest
+	// entry with the same name and value. See above for a definition of "unique id".
+	byNameValue map[pairNameValue]uint64
+}
+
+type pairNameValue struct {
+	name, value string
+}
+
+func (t *headerFieldTable) init() {
+	t.byName = make(map[string]uint64)
+	t.byNameValue = make(map[pairNameValue]uint64)
+}
+
+// len reports the number of entries in the table.
+func (t *headerFieldTable) len() int {
+	return len(t.ents)
+}
+
+// addEntry adds a new entry.
+func (t *headerFieldTable) addEntry(f HeaderField) {
+	id := uint64(t.len()) + t.evictCount + 1
+	t.byName[f.Name] = id
+	t.byNameValue[pairNameValue{f.Name, f.Value}] = id
+	t.ents = append(t.ents, f)
+}
+
+// evictOldest evicts the n oldest entries in the table.
+func (t *headerFieldTable) evictOldest(n int) {
+	if n > t.len() {
+		panic(fmt.Sprintf("evictOldest(%v) on table with %v entries", n, t.len()))
+	}
+	for k := 0; k < n; k++ {
+		f := t.ents[k]
+		id := t.evictCount + uint64(k) + 1
+		if t.byName[f.Name] == id {
+			delete(t.byName, f.Name)
+		}
+		if p := (pairNameValue{f.Name, f.Value}); t.byNameValue[p] == id {
+			delete(t.byNameValue, p)
+		}
+	}
+	copy(t.ents, t.ents[n:])
+	for k := t.len() - n; k < t.len(); k++ {
+		t.ents[k] = HeaderField{} // so strings can be garbage collected
+	}
+	t.ents = t.ents[:t.len()-n]
+	if t.evictCount+uint64(n) < t.evictCount {
+		panic("evictCount overflow")
+	}
+	t.evictCount += uint64(n)
+}
+
+// search finds f in the table. If there is no match, i is 0.
+// If both name and value match, i is the matched index and nameValueMatch
+// becomes true. If only name matches, i points to that index and
+// nameValueMatch becomes false.
+//
+// The returned index is a 1-based HPACK index. For dynamic tables, HPACK says
+// that index 1 should be the newest entry, but t.ents[0] is the oldest entry,
+// meaning t.ents is reversed for dynamic tables. Hence, when t is a dynamic
+// table, the return value i actually refers to the entry t.ents[t.len()-i].
+//
+// All tables are assumed to be a dynamic tables except for the global
+// staticTable pointer.
+//
+// See Section 2.3.3.
+func (t *headerFieldTable) search(f HeaderField) (i uint64, nameValueMatch bool) {
+	if !f.Sensitive {
+		if id := t.byNameValue[pairNameValue{f.Name, f.Value}]; id != 0 {
+			return t.idToIndex(id), true
+		}
+	}
+	if id := t.byName[f.Name]; id != 0 {
+		return t.idToIndex(id), false
+	}
+	return 0, false
+}
+
+// idToIndex converts a unique id to an HPACK index.
+// See Section 2.3.3.
+func (t *headerFieldTable) idToIndex(id uint64) uint64 {
+	if id <= t.evictCount {
+		panic(fmt.Sprintf("id (%v) <= evictCount (%v)", id, t.evictCount))
+	}
+	k := id - t.evictCount - 1 // convert id to an index t.ents[k]
+	if t != staticTable {
+		return uint64(t.len()) - k // dynamic table
+	}
+	return k + 1
+}
+
 func pair(name, value string) HeaderField {
 	return HeaderField{Name: name, Value: value}
 }
 
 // http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-07#appendix-B
-var staticTable = [...]HeaderField{
-	pair(":authority", ""), // index 1 (1-based)
-	pair(":method", "GET"),
-	pair(":method", "POST"),
-	pair(":path", "/"),
-	pair(":path", "/index.html"),
-	pair(":scheme", "http"),
-	pair(":scheme", "https"),
-	pair(":status", "200"),
-	pair(":status", "204"),
-	pair(":status", "206"),
-	pair(":status", "304"),
-	pair(":status", "400"),
-	pair(":status", "404"),
-	pair(":status", "500"),
-	pair("accept-charset", ""),
-	pair("accept-encoding", "gzip, deflate"),
-	pair("accept-language", ""),
-	pair("accept-ranges", ""),
-	pair("accept", ""),
-	pair("access-control-allow-origin", ""),
-	pair("age", ""),
-	pair("allow", ""),
-	pair("authorization", ""),
-	pair("cache-control", ""),
-	pair("content-disposition", ""),
-	pair("content-encoding", ""),
-	pair("content-language", ""),
-	pair("content-length", ""),
-	pair("content-location", ""),
-	pair("content-range", ""),
-	pair("content-type", ""),
-	pair("cookie", ""),
-	pair("date", ""),
-	pair("etag", ""),
-	pair("expect", ""),
-	pair("expires", ""),
-	pair("from", ""),
-	pair("host", ""),
-	pair("if-match", ""),
-	pair("if-modified-since", ""),
-	pair("if-none-match", ""),
-	pair("if-range", ""),
-	pair("if-unmodified-since", ""),
-	pair("last-modified", ""),
-	pair("link", ""),
-	pair("location", ""),
-	pair("max-forwards", ""),
-	pair("proxy-authenticate", ""),
-	pair("proxy-authorization", ""),
-	pair("range", ""),
-	pair("referer", ""),
-	pair("refresh", ""),
-	pair("retry-after", ""),
-	pair("server", ""),
-	pair("set-cookie", ""),
-	pair("strict-transport-security", ""),
-	pair("transfer-encoding", ""),
-	pair("user-agent", ""),
-	pair("vary", ""),
-	pair("via", ""),
-	pair("www-authenticate", ""),
+var staticTable = newStaticTable()
+
+func newStaticTable() *headerFieldTable {
+	t := &headerFieldTable{}
+	t.init()
+	t.addEntry(pair(":authority", ""))
+	t.addEntry(pair(":method", "GET"))
+	t.addEntry(pair(":method", "POST"))
+	t.addEntry(pair(":path", "/"))
+	t.addEntry(pair(":path", "/index.html"))
+	t.addEntry(pair(":scheme", "http"))
+	t.addEntry(pair(":scheme", "https"))
+	t.addEntry(pair(":status", "200"))
+	t.addEntry(pair(":status", "204"))
+	t.addEntry(pair(":status", "206"))
+	t.addEntry(pair(":status", "304"))
+	t.addEntry(pair(":status", "400"))
+	t.addEntry(pair(":status", "404"))
+	t.addEntry(pair(":status", "500"))
+	t.addEntry(pair("accept-charset", ""))
+	t.addEntry(pair("accept-encoding", "gzip, deflate"))
+	t.addEntry(pair("accept-language", ""))
+	t.addEntry(pair("accept-ranges", ""))
+	t.addEntry(pair("accept", ""))
+	t.addEntry(pair("access-control-allow-origin", ""))
+	t.addEntry(pair("age", ""))
+	t.addEntry(pair("allow", ""))
+	t.addEntry(pair("authorization", ""))
+	t.addEntry(pair("cache-control", ""))
+	t.addEntry(pair("content-disposition", ""))
+	t.addEntry(pair("content-encoding", ""))
+	t.addEntry(pair("content-language", ""))
+	t.addEntry(pair("content-length", ""))
+	t.addEntry(pair("content-location", ""))
+	t.addEntry(pair("content-range", ""))
+	t.addEntry(pair("content-type", ""))
+	t.addEntry(pair("cookie", ""))
+	t.addEntry(pair("date", ""))
+	t.addEntry(pair("etag", ""))
+	t.addEntry(pair("expect", ""))
+	t.addEntry(pair("expires", ""))
+	t.addEntry(pair("from", ""))
+	t.addEntry(pair("host", ""))
+	t.addEntry(pair("if-match", ""))
+	t.addEntry(pair("if-modified-since", ""))
+	t.addEntry(pair("if-none-match", ""))
+	t.addEntry(pair("if-range", ""))
+	t.addEntry(pair("if-unmodified-since", ""))
+	t.addEntry(pair("last-modified", ""))
+	t.addEntry(pair("link", ""))
+	t.addEntry(pair("location", ""))
+	t.addEntry(pair("max-forwards", ""))
+	t.addEntry(pair("proxy-authenticate", ""))
+	t.addEntry(pair("proxy-authorization", ""))
+	t.addEntry(pair("range", ""))
+	t.addEntry(pair("referer", ""))
+	t.addEntry(pair("refresh", ""))
+	t.addEntry(pair("retry-after", ""))
+	t.addEntry(pair("server", ""))
+	t.addEntry(pair("set-cookie", ""))
+	t.addEntry(pair("strict-transport-security", ""))
+	t.addEntry(pair("transfer-encoding", ""))
+	t.addEntry(pair("user-agent", ""))
+	t.addEntry(pair("vary", ""))
+	t.addEntry(pair("via", ""))
+	t.addEntry(pair("www-authenticate", ""))
+	return t
 }
 
 var huffmanCodes = [256]uint32{

vendor/golang.org/x/net/http2/server.go

@@ -110,11 +110,38 @@ type Server struct {
 	// activity for the purposes of IdleTimeout.
 	IdleTimeout time.Duration
 
+	// MaxUploadBufferPerConnection is the size of the initial flow
+	// control window for each connections. The HTTP/2 spec does not
+	// allow this to be smaller than 65535 or larger than 2^32-1.
+	// If the value is outside this range, a default value will be
+	// used instead.
+	MaxUploadBufferPerConnection int32
+
+	// MaxUploadBufferPerStream is the size of the initial flow control
+	// window for each stream. The HTTP/2 spec does not allow this to
+	// be larger than 2^32-1. If the value is zero or larger than the
+	// maximum, a default value will be used instead.
+	MaxUploadBufferPerStream int32
+
 	// NewWriteScheduler constructs a write scheduler for a connection.
 	// If nil, a default scheduler is chosen.
 	NewWriteScheduler func() WriteScheduler
 }
 
+func (s *Server) initialConnRecvWindowSize() int32 {
+	if s.MaxUploadBufferPerConnection > initialWindowSize {
+		return s.MaxUploadBufferPerConnection
+	}
+	return 1 << 20
+}
+
+func (s *Server) initialStreamRecvWindowSize() int32 {
+	if s.MaxUploadBufferPerStream > 0 {
+		return s.MaxUploadBufferPerStream
+	}
+	return 1 << 20
+}
+
 func (s *Server) maxReadFrameSize() uint32 {
 	if v := s.MaxReadFrameSize; v >= minMaxFrameSize && v <= maxFrameSize {
 		return v
@@ -255,27 +282,27 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) {
 	defer cancel()
 
 	sc := &serverConn{
-		srv:               s,
-		hs:                opts.baseConfig(),
-		conn:              c,
-		baseCtx:           baseCtx,
-		remoteAddrStr:     c.RemoteAddr().String(),
-		bw:                newBufferedWriter(c),
-		handler:           opts.handler(),
-		streams:           make(map[uint32]*stream),
-		readFrameCh:       make(chan readFrameResult),
-		wantWriteFrameCh:  make(chan FrameWriteRequest, 8),
-		wantStartPushCh:   make(chan startPushRequest, 8),
-		wroteFrameCh:      make(chan frameWriteResult, 1), // buffered; one send in writeFrameAsync
-		bodyReadCh:        make(chan bodyReadMsg),         // buffering doesn't matter either way
-		doneServing:       make(chan struct{}),
-		clientMaxStreams:  math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value"
-		advMaxStreams:     s.maxConcurrentStreams(),
-		initialWindowSize: initialWindowSize,
-		maxFrameSize:      initialMaxFrameSize,
-		headerTableSize:   initialHeaderTableSize,
-		serveG:            newGoroutineLock(),
-		pushEnabled:       true,
+		srv:                         s,
+		hs:                          opts.baseConfig(),
+		conn:                        c,
+		baseCtx:                     baseCtx,
+		remoteAddrStr:               c.RemoteAddr().String(),
+		bw:                          newBufferedWriter(c),
+		handler:                     opts.handler(),
+		streams:                     make(map[uint32]*stream),
+		readFrameCh:                 make(chan readFrameResult),
+		wantWriteFrameCh:            make(chan FrameWriteRequest, 8),
+		wantStartPushCh:             make(chan startPushRequest, 8),
+		wroteFrameCh:                make(chan frameWriteResult, 1), // buffered; one send in writeFrameAsync
+		bodyReadCh:                  make(chan bodyReadMsg),         // buffering doesn't matter either way
+		doneServing:                 make(chan struct{}),
+		clientMaxStreams:            math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value"
+		advMaxStreams:               s.maxConcurrentStreams(),
+		initialStreamSendWindowSize: initialWindowSize,
+		maxFrameSize:                initialMaxFrameSize,
+		headerTableSize:             initialHeaderTableSize,
+		serveG:                      newGoroutineLock(),
+		pushEnabled:                 true,
 	}
 
 	// The net/http package sets the write deadline from the
@@ -294,6 +321,9 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) {
 		sc.writeSched = NewRandomWriteScheduler()
 	}
 
+	// These start at the RFC-specified defaults. If there is a higher
+	// configured value for inflow, that will be updated when we send a
+	// WINDOW_UPDATE shortly after sending SETTINGS.
 	sc.flow.add(initialWindowSize)
 	sc.inflow.add(initialWindowSize)
 	sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf)
@@ -387,34 +417,34 @@ type serverConn struct {
 	writeSched       WriteScheduler
 
 	// Everything following is owned by the serve loop; use serveG.check():
-	serveG                goroutineLock // used to verify funcs are on serve()
-	pushEnabled           bool
-	sawFirstSettings      bool // got the initial SETTINGS frame after the preface
-	needToSendSettingsAck bool
-	unackedSettings       int    // how many SETTINGS have we sent without ACKs?
-	clientMaxStreams      uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit)
-	advMaxStreams         uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client
-	curClientStreams      uint32 // number of open streams initiated by the client
-	curPushedStreams      uint32 // number of open streams initiated by server push
-	maxClientStreamID     uint32 // max ever seen from client (odd), or 0 if there have been no client requests
-	maxPushPromiseID      uint32 // ID of the last push promise (even), or 0 if there have been no pushes
-	streams               map[uint32]*stream
-	initialWindowSize     int32
-	maxFrameSize          int32
-	headerTableSize       uint32
-	peerMaxHeaderListSize uint32            // zero means unknown (default)
-	canonHeader           map[string]string // http2-lower-case -> Go-Canonical-Case
-	writingFrame          bool              // started writing a frame (on serve goroutine or separate)
-	writingFrameAsync     bool              // started a frame on its own goroutine but haven't heard back on wroteFrameCh
-	needsFrameFlush       bool              // last frame write wasn't a flush
-	inGoAway              bool              // we've started to or sent GOAWAY
-	inFrameScheduleLoop   bool              // whether we're in the scheduleFrameWrite loop
-	needToSendGoAway      bool              // we need to schedule a GOAWAY frame write
-	goAwayCode            ErrCode
-	shutdownTimerCh       <-chan time.Time // nil until used
-	shutdownTimer         *time.Timer      // nil until used
-	idleTimer             *time.Timer      // nil if unused
-	idleTimerCh           <-chan time.Time // nil if unused
+	serveG                      goroutineLock // used to verify funcs are on serve()
+	pushEnabled                 bool
+	sawFirstSettings            bool // got the initial SETTINGS frame after the preface
+	needToSendSettingsAck       bool
+	unackedSettings             int    // how many SETTINGS have we sent without ACKs?
+	clientMaxStreams            uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit)
+	advMaxStreams               uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client
+	curClientStreams            uint32 // number of open streams initiated by the client
+	curPushedStreams            uint32 // number of open streams initiated by server push
+	maxClientStreamID           uint32 // max ever seen from client (odd), or 0 if there have been no client requests
+	maxPushPromiseID            uint32 // ID of the last push promise (even), or 0 if there have been no pushes
+	streams                     map[uint32]*stream
+	initialStreamSendWindowSize int32
+	maxFrameSize                int32
+	headerTableSize             uint32
+	peerMaxHeaderListSize       uint32            // zero means unknown (default)
+	canonHeader                 map[string]string // http2-lower-case -> Go-Canonical-Case
+	writingFrame                bool              // started writing a frame (on serve goroutine or separate)
+	writingFrameAsync           bool              // started a frame on its own goroutine but haven't heard back on wroteFrameCh
+	needsFrameFlush             bool              // last frame write wasn't a flush
+	inGoAway                    bool              // we've started to or sent GOAWAY
+	inFrameScheduleLoop         bool              // whether we're in the scheduleFrameWrite loop
+	needToSendGoAway            bool              // we need to schedule a GOAWAY frame write
+	goAwayCode                  ErrCode
+	shutdownTimerCh             <-chan time.Time // nil until used
+	shutdownTimer               *time.Timer      // nil until used
+	idleTimer                   *time.Timer      // nil if unused
+	idleTimerCh                 <-chan time.Time // nil if unused
 
 	// Owned by the writeFrameAsync goroutine:
 	headerWriteBuf bytes.Buffer
@@ -695,15 +725,17 @@ func (sc *serverConn) serve() {
 			{SettingMaxFrameSize, sc.srv.maxReadFrameSize()},
 			{SettingMaxConcurrentStreams, sc.advMaxStreams},
 			{SettingMaxHeaderListSize, sc.maxHeaderListSize()},
-
-			// TODO: more actual settings, notably
-			// SettingInitialWindowSize, but then we also
-			// want to bump up the conn window size the
-			// same amount here right after the settings
+			{SettingInitialWindowSize, uint32(sc.srv.initialStreamRecvWindowSize())},
 		},
 	})
 	sc.unackedSettings++
 
+	// Each connection starts with intialWindowSize inflow tokens.
+	// If a higher value is configured, we add more tokens.
+	if diff := sc.srv.initialConnRecvWindowSize() - initialWindowSize; diff > 0 {
+		sc.sendWindowUpdate(nil, int(diff))
+	}
+
 	if err := sc.readPreface(); err != nil {
 		sc.condlogf(err, "http2: server: error reading preface from client %v: %v", sc.conn.RemoteAddr(), err)
 		return
@@ -1394,9 +1426,9 @@ func (sc *serverConn) processSettingInitialWindowSize(val uint32) error {
 	// adjust the size of all stream flow control windows that it
 	// maintains by the difference between the new value and the
 	// old value."
-	old := sc.initialWindowSize
-	sc.initialWindowSize = int32(val)
-	growth := sc.initialWindowSize - old // may be negative
+	old := sc.initialStreamSendWindowSize
+	sc.initialStreamSendWindowSize = int32(val)
+	growth := int32(val) - old // may be negative
 	for _, st := range sc.streams {
 		if !st.flow.add(growth) {
 			// 6.9.2 Initial Flow Control Window Size
@@ -1718,9 +1750,9 @@ func (sc *serverConn) newStream(id, pusherID uint32, state streamState) *stream
 	}
 	st.cw.Init()
 	st.flow.conn = &sc.flow // link to conn-level counter
-	st.flow.add(sc.initialWindowSize)
-	st.inflow.conn = &sc.inflow      // link to conn-level counter
-	st.inflow.add(initialWindowSize) // TODO: update this when we send a higher initial window size in the initial settings
+	st.flow.add(sc.initialStreamSendWindowSize)
+	st.inflow.conn = &sc.inflow // link to conn-level counter
+	st.inflow.add(sc.srv.initialStreamRecvWindowSize())
 
 	sc.streams[id] = st
 	sc.writeSched.OpenStream(st.id, OpenStreamOptions{PusherID: pusherID})

vendor/golang.org/x/net/idna/idna.go

@@ -1,61 +1,661 @@
-// Copyright 2012 The Go Authors. All rights reserved.
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package idna implements IDNA2008 (Internationalized Domain Names for
-// Applications), defined in RFC 5890, RFC 5891, RFC 5892, RFC 5893 and
-// RFC 5894.
+// Package idna implements IDNA2008 using the compatibility processing
+// defined by UTS (Unicode Technical Standard) #46, which defines a standard to
+// deal with the transition from IDNA2003.
+//
+// IDNA2008 (Internationalized Domain Names for Applications), is defined in RFC
+// 5890, RFC 5891, RFC 5892, RFC 5893 and RFC 5894.
+// UTS #46 is defined in http://www.unicode.org/reports/tr46.
+// See http://unicode.org/cldr/utility/idna.jsp for a visualization of the
+// differences between these two standards.
 package idna // import "golang.org/x/net/idna"
 
 import (
+	"fmt"
 	"strings"
 	"unicode/utf8"
+
+	"golang.org/x/text/secure/bidirule"
+	"golang.org/x/text/unicode/norm"
 )
 
-// TODO(nigeltao): specify when errors occur. For example, is ToASCII(".") or
-// ToASCII("foo\x00") an error? See also http://www.unicode.org/faq/idn.html#11
+// NOTE: Unlike common practice in Go APIs, the functions will return a
+// sanitized domain name in case of errors. Browsers sometimes use a partially
+// evaluated string as lookup.
+// TODO: the current error handling is, in my opinion, the least opinionated.
+// Other strategies are also viable, though:
+// Option 1) Return an empty string in case of error, but allow the user to
+//    specify explicitly which errors to ignore.
+// Option 2) Return the partially evaluated string if it is itself a valid
+//    string, otherwise return the empty string in case of error.
+// Option 3) Option 1 and 2.
+// Option 4) Always return an empty string for now and implement Option 1 as
+//    needed, and document that the return string may not be empty in case of
+//    error in the future.
+// I think Option 1 is best, but it is quite opinionated.
 
-// acePrefix is the ASCII Compatible Encoding prefix.
-const acePrefix = "xn--"
+// ToASCII is a wrapper for Punycode.ToASCII.
+func ToASCII(s string) (string, error) {
+	return Punycode.process(s, true)
+}
+
+// ToUnicode is a wrapper for Punycode.ToUnicode.
+func ToUnicode(s string) (string, error) {
+	return Punycode.process(s, false)
+}
+
+// An Option configures a Profile at creation time.
+type Option func(*options)
+
+// Transitional sets a Profile to use the Transitional mapping as defined in UTS
+// #46. This will cause, for example, "ß" to be mapped to "ss". Using the
+// transitional mapping provides a compromise between IDNA2003 and IDNA2008
+// compatibility. It is used by most browsers when resolving domain names. This
+// option is only meaningful if combined with MapForLookup.
+func Transitional(transitional bool) Option {
+	return func(o *options) { o.transitional = true }
+}
+
+// VerifyDNSLength sets whether a Profile should fail if any of the IDN parts
+// are longer than allowed by the RFC.
+func VerifyDNSLength(verify bool) Option {
+	return func(o *options) { o.verifyDNSLength = verify }
+}
+
+// ValidateLabels sets whether to check the mandatory label validation criteria
+// as defined in Section 5.4 of RFC 5891. This includes testing for correct use
+// of hyphens ('-'), normalization, validity of runes, and the context rules.
+func ValidateLabels(enable bool) Option {
+	return func(o *options) {
+		// Don't override existing mappings, but set one that at least checks
+		// normalization if it is not set.
+		if o.mapping == nil && enable {
+			o.mapping = normalize
+		}
+		o.trie = trie
+		o.validateLabels = enable
+		o.fromPuny = validateFromPunycode
+	}
+}
+
+// StrictDomainName limits the set of permissable ASCII characters to those
+// allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the
+// hyphen). This is set by default for MapForLookup and ValidateForRegistration.
+//
+// This option is useful, for instance, for browsers that allow characters
+// outside this range, for example a '_' (U+005F LOW LINE). See
+// http://www.rfc-editor.org/std/std3.txt for more details This option
+// corresponds to the UseSTD3ASCIIRules option in UTS #46.
+func StrictDomainName(use bool) Option {
+	return func(o *options) {
+		o.trie = trie
+		o.useSTD3Rules = use
+		o.fromPuny = validateFromPunycode
+	}
+}
+
+// NOTE: the following options pull in tables. The tables should not be linked
+// in as long as the options are not used.
+
+// BidiRule enables the Bidi rule as defined in RFC 5893. Any application
+// that relies on proper validation of labels should include this rule.
+func BidiRule() Option {
+	return func(o *options) { o.bidirule = bidirule.ValidString }
+}
+
+// ValidateForRegistration sets validation options to verify that a given IDN is
+// properly formatted for registration as defined by Section 4 of RFC 5891.
+func ValidateForRegistration() Option {
+	return func(o *options) {
+		o.mapping = validateRegistration
+		StrictDomainName(true)(o)
+		ValidateLabels(true)(o)
+		VerifyDNSLength(true)(o)
+		BidiRule()(o)
+	}
+}
+
+// MapForLookup sets validation and mapping options such that a given IDN is
+// transformed for domain name lookup according to the requirements set out in
+// Section 5 of RFC 5891. The mappings follow the recommendations of RFC 5894,
+// RFC 5895 and UTS 46. It does not add the Bidi Rule. Use the BidiRule option
+// to add this check.
+//
+// The mappings include normalization and mapping case, width and other
+// compatibility mappings.
+func MapForLookup() Option {
+	return func(o *options) {
+		o.mapping = validateAndMap
+		StrictDomainName(true)(o)
+		ValidateLabels(true)(o)
+	}
+}
+
+type options struct {
+	transitional    bool
+	useSTD3Rules    bool
+	validateLabels  bool
+	verifyDNSLength bool
+
+	trie *idnaTrie
+
+	// fromPuny calls validation rules when converting A-labels to U-labels.
+	fromPuny func(p *Profile, s string) error
+
+	// mapping implements a validation and mapping step as defined in RFC 5895
+	// or UTS 46, tailored to, for example, domain registration or lookup.
+	mapping func(p *Profile, s string) (string, error)
+
+	// bidirule, if specified, checks whether s conforms to the Bidi Rule
+	// defined in RFC 5893.
+	bidirule func(s string) bool
+}
+
+// A Profile defines the configuration of a IDNA mapper.
+type Profile struct {
+	options
+}
+
+func apply(o *options, opts []Option) {
+	for _, f := range opts {
+		f(o)
+	}
+}
+
+// New creates a new Profile.
+//
+// With no options, the returned Profile is the most permissive and equals the
+// Punycode Profile. Options can be passed to further restrict the Profile. The
+// MapForLookup and ValidateForRegistration options set a collection of options,
+// for lookup and registration purposes respectively, which can be tailored by
+// adding more fine-grained options, where later options override earlier
+// options.
+func New(o ...Option) *Profile {
+	p := &Profile{}
+	apply(&p.options, o)
+	return p
+}
 
 // ToASCII converts a domain or domain label to its ASCII form. For example,
 // ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and
-// ToASCII("golang") is "golang".
-func ToASCII(s string) (string, error) {
-	if ascii(s) {
-		return s, nil
-	}
-	labels := strings.Split(s, ".")
-	for i, label := range labels {
-		if !ascii(label) {
-			a, err := encode(acePrefix, label)
-			if err != nil {
-				return "", err
-			}
-			labels[i] = a
-		}
-	}
-	return strings.Join(labels, "."), nil
+// ToASCII("golang") is "golang". If an error is encountered it will return
+// an error and a (partially) processed result.
+func (p *Profile) ToASCII(s string) (string, error) {
+	return p.process(s, true)
 }
 
 // ToUnicode converts a domain or domain label to its Unicode form. For example,
 // ToUnicode("xn--bcher-kva.example.com") is "bücher.example.com", and
-// ToUnicode("golang") is "golang".
-func ToUnicode(s string) (string, error) {
-	if !strings.Contains(s, acePrefix) {
-		return s, nil
+// ToUnicode("golang") is "golang". If an error is encountered it will return
+// an error and a (partially) processed result.
+func (p *Profile) ToUnicode(s string) (string, error) {
+	pp := *p
+	pp.transitional = false
+	return pp.process(s, false)
+}
+
+// String reports a string with a description of the profile for debugging
+// purposes. The string format may change with different versions.
+func (p *Profile) String() string {
+	s := ""
+	if p.transitional {
+		s = "Transitional"
+	} else {
+		s = "NonTransitional"
 	}
-	labels := strings.Split(s, ".")
-	for i, label := range labels {
-		if strings.HasPrefix(label, acePrefix) {
-			u, err := decode(label[len(acePrefix):])
-			if err != nil {
-				return "", err
+	if p.useSTD3Rules {
+		s += ":UseSTD3Rules"
+	}
+	if p.validateLabels {
+		s += ":ValidateLabels"
+	}
+	if p.verifyDNSLength {
+		s += ":VerifyDNSLength"
+	}
+	return s
+}
+
+var (
+	// Punycode is a Profile that does raw punycode processing with a minimum
+	// of validation.
+	Punycode *Profile = punycode
+
+	// Lookup is the recommended profile for looking up domain names, according
+	// to Section 5 of RFC 5891. The exact configuration of this profile may
+	// change over time.
+	Lookup *Profile = lookup
+
+	// Display is the recommended profile for displaying domain names.
+	// The configuration of this profile may change over time.
+	Display *Profile = display
+
+	// Registration is the recommended profile for checking whether a given
+	// IDN is valid for registration, according to Section 4 of RFC 5891.
+	Registration *Profile = registration
+
+	punycode = &Profile{}
+	lookup   = &Profile{options{
+		transitional:   true,
+		useSTD3Rules:   true,
+		validateLabels: true,
+		trie:           trie,
+		fromPuny:       validateFromPunycode,
+		mapping:        validateAndMap,
+		bidirule:       bidirule.ValidString,
+	}}
+	display = &Profile{options{
+		useSTD3Rules:   true,
+		validateLabels: true,
+		trie:           trie,
+		fromPuny:       validateFromPunycode,
+		mapping:        validateAndMap,
+		bidirule:       bidirule.ValidString,
+	}}
+	registration = &Profile{options{
+		useSTD3Rules:    true,
+		validateLabels:  true,
+		verifyDNSLength: true,
+		trie:            trie,
+		fromPuny:        validateFromPunycode,
+		mapping:         validateRegistration,
+		bidirule:        bidirule.ValidString,
+	}}
+
+	// TODO: profiles
+	// Register: recommended for approving domain names: don't do any mappings
+	// but rather reject on invalid input. Bundle or block deviation characters.
+)
+
+type labelError struct{ label, code_ string }
+
+func (e labelError) code() string { return e.code_ }
+func (e labelError) Error() string {
+	return fmt.Sprintf("idna: invalid label %q", e.label)
+}
+
+type runeError rune
+
+func (e runeError) code() string { return "P1" }
+func (e runeError) Error() string {
+	return fmt.Sprintf("idna: disallowed rune %U", e)
+}
+
+// process implements the algorithm described in section 4 of UTS #46,
+// see http://www.unicode.org/reports/tr46.
+func (p *Profile) process(s string, toASCII bool) (string, error) {
+	var err error
+	if p.mapping != nil {
+		s, err = p.mapping(p, s)
+	}
+	// Remove leading empty labels.
+	for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
+	}
+	// It seems like we should only create this error on ToASCII, but the
+	// UTS 46 conformance tests suggests we should always check this.
+	if err == nil && p.verifyDNSLength && s == "" {
+		err = &labelError{s, "A4"}
+	}
+	labels := labelIter{orig: s}
+	for ; !labels.done(); labels.next() {
+		label := labels.label()
+		if label == "" {
+			// Empty labels are not okay. The label iterator skips the last
+			// label if it is empty.
+			if err == nil && p.verifyDNSLength {
+				err = &labelError{s, "A4"}
 			}
-			labels[i] = u
+			continue
+		}
+		if strings.HasPrefix(label, acePrefix) {
+			u, err2 := decode(label[len(acePrefix):])
+			if err2 != nil {
+				if err == nil {
+					err = err2
+				}
+				// Spec says keep the old label.
+				continue
+			}
+			labels.set(u)
+			if err == nil && p.validateLabels {
+				err = p.fromPuny(p, u)
+			}
+			if err == nil {
+				// This should be called on NonTransitional, according to the
+				// spec, but that currently does not have any effect. Use the
+				// original profile to preserve options.
+				err = p.validateLabel(u)
+			}
+		} else if err == nil {
+			err = p.validateLabel(label)
 		}
 	}
-	return strings.Join(labels, "."), nil
+	if toASCII {
+		for labels.reset(); !labels.done(); labels.next() {
+			label := labels.label()
+			if !ascii(label) {
+				a, err2 := encode(acePrefix, label)
+				if err == nil {
+					err = err2
+				}
+				label = a
+				labels.set(a)
+			}
+			n := len(label)
+			if p.verifyDNSLength && err == nil && (n == 0 || n > 63) {
+				err = &labelError{label, "A4"}
+			}
+		}
+	}
+	s = labels.result()
+	if toASCII && p.verifyDNSLength && err == nil {
+		// Compute the length of the domain name minus the root label and its dot.
+		n := len(s)
+		if n > 0 && s[n-1] == '.' {
+			n--
+		}
+		if len(s) < 1 || n > 253 {
+			err = &labelError{s, "A4"}
+		}
+	}
+	return s, err
+}
+
+func normalize(p *Profile, s string) (string, error) {
+	return norm.NFC.String(s), nil
+}
+
+func validateRegistration(p *Profile, s string) (string, error) {
+	if !norm.NFC.IsNormalString(s) {
+		return s, &labelError{s, "V1"}
+	}
+	var err error
+	for i := 0; i < len(s); {
+		v, sz := trie.lookupString(s[i:])
+		i += sz
+		// Copy bytes not copied so far.
+		switch p.simplify(info(v).category()) {
+		// TODO: handle the NV8 defined in the Unicode idna data set to allow
+		// for strict conformance to IDNA2008.
+		case valid, deviation:
+		case disallowed, mapped, unknown, ignored:
+			if err == nil {
+				r, _ := utf8.DecodeRuneInString(s[i:])
+				err = runeError(r)
+			}
+		}
+	}
+	return s, err
+}
+
+func validateAndMap(p *Profile, s string) (string, error) {
+	var (
+		err error
+		b   []byte
+		k   int
+	)
+	for i := 0; i < len(s); {
+		v, sz := trie.lookupString(s[i:])
+		start := i
+		i += sz
+		// Copy bytes not copied so far.
+		switch p.simplify(info(v).category()) {
+		case valid:
+			continue
+		case disallowed:
+			if err == nil {
+				r, _ := utf8.DecodeRuneInString(s[i:])
+				err = runeError(r)
+			}
+			continue
+		case mapped, deviation:
+			b = append(b, s[k:start]...)
+			b = info(v).appendMapping(b, s[start:i])
+		case ignored:
+			b = append(b, s[k:start]...)
+			// drop the rune
+		case unknown:
+			b = append(b, s[k:start]...)
+			b = append(b, "\ufffd"...)
+		}
+		k = i
+	}
+	if k == 0 {
+		// No changes so far.
+		s = norm.NFC.String(s)
+	} else {
+		b = append(b, s[k:]...)
+		if norm.NFC.QuickSpan(b) != len(b) {
+			b = norm.NFC.Bytes(b)
+		}
+		// TODO: the punycode converters require strings as input.
+		s = string(b)
+	}
+	return s, err
+}
+
+// A labelIter allows iterating over domain name labels.
+type labelIter struct {
+	orig     string
+	slice    []string
+	curStart int
+	curEnd   int
+	i        int
+}
+
+func (l *labelIter) reset() {
+	l.curStart = 0
+	l.curEnd = 0
+	l.i = 0
+}
+
+func (l *labelIter) done() bool {
+	return l.curStart >= len(l.orig)
+}
+
+func (l *labelIter) result() string {
+	if l.slice != nil {
+		return strings.Join(l.slice, ".")
+	}
+	return l.orig
+}
+
+func (l *labelIter) label() string {
+	if l.slice != nil {
+		return l.slice[l.i]
+	}
+	p := strings.IndexByte(l.orig[l.curStart:], '.')
+	l.curEnd = l.curStart + p
+	if p == -1 {
+		l.curEnd = len(l.orig)
+	}
+	return l.orig[l.curStart:l.curEnd]
+}
+
+// next sets the value to the next label. It skips the last label if it is empty.
+func (l *labelIter) next() {
+	l.i++
+	if l.slice != nil {
+		if l.i >= len(l.slice) || l.i == len(l.slice)-1 && l.slice[l.i] == "" {
+			l.curStart = len(l.orig)
+		}
+	} else {
+		l.curStart = l.curEnd + 1
+		if l.curStart == len(l.orig)-1 && l.orig[l.curStart] == '.' {
+			l.curStart = len(l.orig)
+		}
+	}
+}
+
+func (l *labelIter) set(s string) {
+	if l.slice == nil {
+		l.slice = strings.Split(l.orig, ".")
+	}
+	l.slice[l.i] = s
+}
+
+// acePrefix is the ASCII Compatible Encoding prefix.
+const acePrefix = "xn--"
+
+func (p *Profile) simplify(cat category) category {
+	switch cat {
+	case disallowedSTD3Mapped:
+		if p.useSTD3Rules {
+			cat = disallowed
+		} else {
+			cat = mapped
+		}
+	case disallowedSTD3Valid:
+		if p.useSTD3Rules {
+			cat = disallowed
+		} else {
+			cat = valid
+		}
+	case deviation:
+		if !p.transitional {
+			cat = valid
+		}
+	case validNV8, validXV8:
+		// TODO: handle V2008
+		cat = valid
+	}
+	return cat
+}
+
+func validateFromPunycode(p *Profile, s string) error {
+	if !norm.NFC.IsNormalString(s) {
+		return &labelError{s, "V1"}
+	}
+	for i := 0; i < len(s); {
+		v, sz := trie.lookupString(s[i:])
+		if c := p.simplify(info(v).category()); c != valid && c != deviation {
+			return &labelError{s, "V6"}
+		}
+		i += sz
+	}
+	return nil
+}
+
+const (
+	zwnj = "\u200c"
+	zwj  = "\u200d"
+)
+
+type joinState int8
+
+const (
+	stateStart joinState = iota
+	stateVirama
+	stateBefore
+	stateBeforeVirama
+	stateAfter
+	stateFAIL
+)
+
+var joinStates = [][numJoinTypes]joinState{
+	stateStart: {
+		joiningL:   stateBefore,
+		joiningD:   stateBefore,
+		joinZWNJ:   stateFAIL,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateVirama,
+	},
+	stateVirama: {
+		joiningL: stateBefore,
+		joiningD: stateBefore,
+	},
+	stateBefore: {
+		joiningL:   stateBefore,
+		joiningD:   stateBefore,
+		joiningT:   stateBefore,
+		joinZWNJ:   stateAfter,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateBeforeVirama,
+	},
+	stateBeforeVirama: {
+		joiningL: stateBefore,
+		joiningD: stateBefore,
+		joiningT: stateBefore,
+	},
+	stateAfter: {
+		joiningL:   stateFAIL,
+		joiningD:   stateBefore,
+		joiningT:   stateAfter,
+		joiningR:   stateStart,
+		joinZWNJ:   stateFAIL,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateAfter, // no-op as we can't accept joiners here
+	},
+	stateFAIL: {
+		0:          stateFAIL,
+		joiningL:   stateFAIL,
+		joiningD:   stateFAIL,
+		joiningT:   stateFAIL,
+		joiningR:   stateFAIL,
+		joinZWNJ:   stateFAIL,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateFAIL,
+	},
+}
+
+// validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are
+// already implicitly satisfied by the overall implementation.
+func (p *Profile) validateLabel(s string) error {
+	if s == "" {
+		if p.verifyDNSLength {
+			return &labelError{s, "A4"}
+		}
+		return nil
+	}
+	if p.bidirule != nil && !p.bidirule(s) {
+		return &labelError{s, "B"}
+	}
+	if !p.validateLabels {
+		return nil
+	}
+	trie := p.trie // p.validateLabels is only set if trie is set.
+	if len(s) > 4 && s[2] == '-' && s[3] == '-' {
+		return &labelError{s, "V2"}
+	}
+	if s[0] == '-' || s[len(s)-1] == '-' {
+		return &labelError{s, "V3"}
+	}
+	// TODO: merge the use of this in the trie.
+	v, sz := trie.lookupString(s)
+	x := info(v)
+	if x.isModifier() {
+		return &labelError{s, "V5"}
+	}
+	// Quickly return in the absence of zero-width (non) joiners.
+	if strings.Index(s, zwj) == -1 && strings.Index(s, zwnj) == -1 {
+		return nil
+	}
+	st := stateStart
+	for i := 0; ; {
+		jt := x.joinType()
+		if s[i:i+sz] == zwj {
+			jt = joinZWJ
+		} else if s[i:i+sz] == zwnj {
+			jt = joinZWNJ
+		}
+		st = joinStates[st][jt]
+		if x.isViramaModifier() {
+			st = joinStates[st][joinVirama]
+		}
+		if i += sz; i == len(s) {
+			break
+		}
+		v, sz = trie.lookupString(s[i:])
+		x = info(v)
+	}
+	if st == stateFAIL || st == stateAfter {
+		return &labelError{s, "C"}
+	}
+	return nil
 }
 
 func ascii(s string) bool {

vendor/golang.org/x/net/idna/punycode.go

@@ -1,4 +1,6 @@
-// Copyright 2012 The Go Authors. All rights reserved.
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -7,7 +9,6 @@ package idna
 // This file implements the Punycode algorithm from RFC 3492.
 
 import (
-	"fmt"
 	"math"
 	"strings"
 	"unicode/utf8"
@@ -27,6 +28,8 @@ const (
 	tmin        int32 = 1
 )
 
+func punyError(s string) error { return &labelError{s, "A3"} }
+
 // decode decodes a string as specified in section 6.2.
 func decode(encoded string) (string, error) {
 	if encoded == "" {
@@ -34,7 +37,7 @@ func decode(encoded string) (string, error) {
 	}
 	pos := 1 + strings.LastIndex(encoded, "-")
 	if pos == 1 {
-		return "", fmt.Errorf("idna: invalid label %q", encoded)
+		return "", punyError(encoded)
 	}
 	if pos == len(encoded) {
 		return encoded[:len(encoded)-1], nil
@@ -50,16 +53,16 @@ func decode(encoded string) (string, error) {
 		oldI, w := i, int32(1)
 		for k := base; ; k += base {
 			if pos == len(encoded) {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 			digit, ok := decodeDigit(encoded[pos])
 			if !ok {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 			pos++
 			i += digit * w
 			if i < 0 {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 			t := k - bias
 			if t < tmin {
@@ -72,7 +75,7 @@ func decode(encoded string) (string, error) {
 			}
 			w *= base - t
 			if w >= math.MaxInt32/base {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 		}
 		x := int32(len(output) + 1)
@@ -80,7 +83,7 @@ func decode(encoded string) (string, error) {
 		n += i / x
 		i %= x
 		if n > utf8.MaxRune || len(output) >= 1024 {
-			return "", fmt.Errorf("idna: invalid label %q", encoded)
+			return "", punyError(encoded)
 		}
 		output = append(output, 0)
 		copy(output[i+1:], output[i:])
@@ -121,14 +124,14 @@ func encode(prefix, s string) (string, error) {
 		}
 		delta += (m - n) * (h + 1)
 		if delta < 0 {
-			return "", fmt.Errorf("idna: invalid label %q", s)
+			return "", punyError(s)
 		}
 		n = m
 		for _, r := range s {
 			if r < n {
 				delta++
 				if delta < 0 {
-					return "", fmt.Errorf("idna: invalid label %q", s)
+					return "", punyError(s)
 				}
 				continue
 			}

vendor/golang.org/x/net/idna/trie.go

@@ -0,0 +1,72 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package idna
+
+// appendMapping appends the mapping for the respective rune. isMapped must be
+// true. A mapping is a categorization of a rune as defined in UTS #46.
+func (c info) appendMapping(b []byte, s string) []byte {
+	index := int(c >> indexShift)
+	if c&xorBit == 0 {
+		s := mappings[index:]
+		return append(b, s[1:s[0]+1]...)
+	}
+	b = append(b, s...)
+	if c&inlineXOR == inlineXOR {
+		// TODO: support and handle two-byte inline masks
+		b[len(b)-1] ^= byte(index)
+	} else {
+		for p := len(b) - int(xorData[index]); p < len(b); p++ {
+			index++
+			b[p] ^= xorData[index]
+		}
+	}
+	return b
+}
+
+// Sparse block handling code.
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type sparseBlocks struct {
+	values []valueRange
+	offset []uint16
+}
+
+var idnaSparse = sparseBlocks{
+	values: idnaSparseValues[:],
+	offset: idnaSparseOffset[:],
+}
+
+// Don't use newIdnaTrie to avoid unconditional linking in of the table.
+var trie = &idnaTrie{}
+
+// lookup determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	offset := t.offset[n]
+	header := t.values[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}

vendor/golang.org/x/net/idna/trieval.go

@@ -0,0 +1,114 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package idna
+
+// This file contains definitions for interpreting the trie value of the idna
+// trie generated by "go run gen*.go". It is shared by both the generator
+// program and the resultant package. Sharing is achieved by the generator
+// copying gen_trieval.go to trieval.go and changing what's above this comment.
+
+// info holds information from the IDNA mapping table for a single rune. It is
+// the value returned by a trie lookup. In most cases, all information fits in
+// a 16-bit value. For mappings, this value may contain an index into a slice
+// with the mapped string. Such mappings can consist of the actual mapped value
+// or an XOR pattern to be applied to the bytes of the UTF8 encoding of the
+// input rune. This technique is used by the cases packages and reduces the
+// table size significantly.
+//
+// The per-rune values have the following format:
+//
+//   if mapped {
+//     if inlinedXOR {
+//       15..13 inline XOR marker
+//       12..11 unused
+//       10..3  inline XOR mask
+//     } else {
+//       15..3  index into xor or mapping table
+//     }
+//   } else {
+//       15..13 unused
+//           12 modifier (including virama)
+//           11 virama modifier
+//       10..8  joining type
+//        7..3  category type
+//   }
+//      2  use xor pattern
+//   1..0  mapped category
+//
+// See the definitions below for a more detailed description of the various
+// bits.
+type info uint16
+
+const (
+	catSmallMask = 0x3
+	catBigMask   = 0xF8
+	indexShift   = 3
+	xorBit       = 0x4    // interpret the index as an xor pattern
+	inlineXOR    = 0xE000 // These bits are set if the XOR pattern is inlined.
+
+	joinShift = 8
+	joinMask  = 0x07
+
+	viramaModifier = 0x0800
+	modifier       = 0x1000
+)
+
+// A category corresponds to a category defined in the IDNA mapping table.
+type category uint16
+
+const (
+	unknown              category = 0 // not defined currently in unicode.
+	mapped               category = 1
+	disallowedSTD3Mapped category = 2
+	deviation            category = 3
+)
+
+const (
+	valid               category = 0x08
+	validNV8            category = 0x18
+	validXV8            category = 0x28
+	disallowed          category = 0x40
+	disallowedSTD3Valid category = 0x80
+	ignored             category = 0xC0
+)
+
+// join types and additional rune information
+const (
+	joiningL = (iota + 1)
+	joiningD
+	joiningT
+	joiningR
+
+	//the following types are derived during processing
+	joinZWJ
+	joinZWNJ
+	joinVirama
+	numJoinTypes
+)
+
+func (c info) isMapped() bool {
+	return c&0x3 != 0
+}
+
+func (c info) category() category {
+	small := c & catSmallMask
+	if small != 0 {
+		return category(small)
+	}
+	return category(c & catBigMask)
+}
+
+func (c info) joinType() info {
+	if c.isMapped() {
+		return 0
+	}
+	return (c >> joinShift) & joinMask
+}
+
+func (c info) isModifier() bool {
+	return c&(modifier|catSmallMask) == modifier
+}
+
+func (c info) isViramaModifier() bool {
+	return c&(viramaModifier|catSmallMask) == viramaModifier
+}

vendor/golang.org/x/text/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/golang.org/x/text/PATENTS

@@ -0,0 +1,22 @@
+Additional IP Rights Grant (Patents)
+
+"This implementation" means the copyrightable works distributed by
+Google as part of the Go project.
+
+Google hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section)
+patent license to make, have made, use, offer to sell, sell, import,
+transfer and otherwise run, modify and propagate the contents of this
+implementation of Go, where such license applies only to those patent
+claims, both currently owned or controlled by Google and acquired in
+the future, licensable by Google that are necessarily infringed by this
+implementation of Go.  This grant does not include claims that would be
+infringed only as a consequence of further modification of this
+implementation.  If you or your agent or exclusive licensee institute or
+order or agree to the institution of patent litigation against any
+entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that this implementation of Go or any code incorporated within this
+implementation of Go constitutes direct or contributory patent
+infringement, or inducement of patent infringement, then any patent
+rights granted to you under this License for this implementation of Go
+shall terminate as of the date such litigation is filed.

vendor/golang.org/x/text/README

@@ -0,0 +1,23 @@
+This repository holds supplementary Go libraries for text processing, many involving Unicode.
+
+To submit changes to this repository, see http://golang.org/doc/contribute.html.
+
+To generate the tables in this repository (except for the encoding tables),
+run go generate from this directory. By default tables are generated for the
+Unicode version in core and the CLDR version defined in
+golang.org/x/text/unicode/cldr.
+
+Running go generate will as a side effect create a DATA subdirectory in this
+directory which holds all files that are used as a source for generating the
+tables. This directory will also serve as a cache.
+
+Run
+
+	go test ./...
+
+from this directory to run all tests. Add the "-tags icu" flag to also run
+ICU conformance tests (if available). This requires that you have the correct
+ICU version installed on your system.
+
+TODO:
+- updating unversioned source files.

vendor/golang.org/x/text/internal/gen/code.go

@@ -0,0 +1,351 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gen
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"hash"
+	"hash/fnv"
+	"io"
+	"log"
+	"os"
+	"reflect"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// This file contains utilities for generating code.
+
+// TODO: other write methods like:
+// - slices, maps, types, etc.
+
+// CodeWriter is a utility for writing structured code. It computes the content
+// hash and size of written content. It ensures there are newlines between
+// written code blocks.
+type CodeWriter struct {
+	buf  bytes.Buffer
+	Size int
+	Hash hash.Hash32 // content hash
+	gob  *gob.Encoder
+	// For comments we skip the usual one-line separator if they are followed by
+	// a code block.
+	skipSep bool
+}
+
+func (w *CodeWriter) Write(p []byte) (n int, err error) {
+	return w.buf.Write(p)
+}
+
+// NewCodeWriter returns a new CodeWriter.
+func NewCodeWriter() *CodeWriter {
+	h := fnv.New32()
+	return &CodeWriter{Hash: h, gob: gob.NewEncoder(h)}
+}
+
+// WriteGoFile appends the buffer with the total size of all created structures
+// and writes it as a Go file to the the given file with the given package name.
+func (w *CodeWriter) WriteGoFile(filename, pkg string) {
+	f, err := os.Create(filename)
+	if err != nil {
+		log.Fatalf("Could not create file %s: %v", filename, err)
+	}
+	defer f.Close()
+	if _, err = w.WriteGo(f, pkg); err != nil {
+		log.Fatalf("Error writing file %s: %v", filename, err)
+	}
+}
+
+// WriteGo appends the buffer with the total size of all created structures and
+// writes it as a Go file to the the given writer with the given package name.
+func (w *CodeWriter) WriteGo(out io.Writer, pkg string) (n int, err error) {
+	sz := w.Size
+	w.WriteComment("Total table size %d bytes (%dKiB); checksum: %X\n", sz, sz/1024, w.Hash.Sum32())
+	defer w.buf.Reset()
+	return WriteGo(out, pkg, w.buf.Bytes())
+}
+
+func (w *CodeWriter) printf(f string, x ...interface{}) {
+	fmt.Fprintf(w, f, x...)
+}
+
+func (w *CodeWriter) insertSep() {
+	if w.skipSep {
+		w.skipSep = false
+		return
+	}
+	// Use at least two newlines to ensure a blank space between the previous
+	// block. WriteGoFile will remove extraneous newlines.
+	w.printf("\n\n")
+}
+
+// WriteComment writes a comment block. All line starts are prefixed with "//".
+// Initial empty lines are gobbled. The indentation for the first line is
+// stripped from consecutive lines.
+func (w *CodeWriter) WriteComment(comment string, args ...interface{}) {
+	s := fmt.Sprintf(comment, args...)
+	s = strings.Trim(s, "\n")
+
+	// Use at least two newlines to ensure a blank space between the previous
+	// block. WriteGoFile will remove extraneous newlines.
+	w.printf("\n\n// ")
+	w.skipSep = true
+
+	// strip first indent level.
+	sep := "\n"
+	for ; len(s) > 0 && (s[0] == '\t' || s[0] == ' '); s = s[1:] {
+		sep += s[:1]
+	}
+
+	strings.NewReplacer(sep, "\n// ", "\n", "\n// ").WriteString(w, s)
+
+	w.printf("\n")
+}
+
+func (w *CodeWriter) writeSizeInfo(size int) {
+	w.printf("// Size: %d bytes\n", size)
+}
+
+// WriteConst writes a constant of the given name and value.
+func (w *CodeWriter) WriteConst(name string, x interface{}) {
+	w.insertSep()
+	v := reflect.ValueOf(x)
+
+	switch v.Type().Kind() {
+	case reflect.String:
+		w.printf("const %s %s = ", name, typeName(x))
+		w.WriteString(v.String())
+		w.printf("\n")
+	default:
+		w.printf("const %s = %#v\n", name, x)
+	}
+}
+
+// WriteVar writes a variable of the given name and value.
+func (w *CodeWriter) WriteVar(name string, x interface{}) {
+	w.insertSep()
+	v := reflect.ValueOf(x)
+	oldSize := w.Size
+	sz := int(v.Type().Size())
+	w.Size += sz
+
+	switch v.Type().Kind() {
+	case reflect.String:
+		w.printf("var %s %s = ", name, typeName(x))
+		w.WriteString(v.String())
+	case reflect.Struct:
+		w.gob.Encode(x)
+		fallthrough
+	case reflect.Slice, reflect.Array:
+		w.printf("var %s = ", name)
+		w.writeValue(v)
+		w.writeSizeInfo(w.Size - oldSize)
+	default:
+		w.printf("var %s %s = ", name, typeName(x))
+		w.gob.Encode(x)
+		w.writeValue(v)
+		w.writeSizeInfo(w.Size - oldSize)
+	}
+	w.printf("\n")
+}
+
+func (w *CodeWriter) writeValue(v reflect.Value) {
+	x := v.Interface()
+	switch v.Kind() {
+	case reflect.String:
+		w.WriteString(v.String())
+	case reflect.Array:
+		// Don't double count: callers of WriteArray count on the size being
+		// added, so we need to discount it here.
+		w.Size -= int(v.Type().Size())
+		w.writeSlice(x, true)
+	case reflect.Slice:
+		w.writeSlice(x, false)
+	case reflect.Struct:
+		w.printf("%s{\n", typeName(v.Interface()))
+		t := v.Type()
+		for i := 0; i < v.NumField(); i++ {
+			w.printf("%s: ", t.Field(i).Name)
+			w.writeValue(v.Field(i))
+			w.printf(",\n")
+		}
+		w.printf("}")
+	default:
+		w.printf("%#v", x)
+	}
+}
+
+// WriteString writes a string literal.
+func (w *CodeWriter) WriteString(s string) {
+	s = strings.Replace(s, `\`, `\\`, -1)
+	io.WriteString(w.Hash, s) // content hash
+	w.Size += len(s)
+
+	const maxInline = 40
+	if len(s) <= maxInline {
+		w.printf("%q", s)
+		return
+	}
+
+	// We will render the string as a multi-line string.
+	const maxWidth = 80 - 4 - len(`"`) - len(`" +`)
+
+	// When starting on its own line, go fmt indents line 2+ an extra level.
+	n, max := maxWidth, maxWidth-4
+
+	// As per https://golang.org/issue/18078, the compiler has trouble
+	// compiling the concatenation of many strings, s0 + s1 + s2 + ... + sN,
+	// for large N. We insert redundant, explicit parentheses to work around
+	// that, lowering the N at any given step: (s0 + s1 + ... + s63) + (s64 +
+	// ... + s127) + etc + (etc + ... + sN).
+	explicitParens, extraComment := len(s) > 128*1024, ""
+	if explicitParens {
+		w.printf(`(`)
+		extraComment = "; the redundant, explicit parens are for https://golang.org/issue/18078"
+	}
+
+	// Print "" +\n, if a string does not start on its own line.
+	b := w.buf.Bytes()
+	if p := len(bytes.TrimRight(b, " \t")); p > 0 && b[p-1] != '\n' {
+		w.printf("\"\" + // Size: %d bytes%s\n", len(s), extraComment)
+		n, max = maxWidth, maxWidth
+	}
+
+	w.printf(`"`)
+
+	for sz, p, nLines := 0, 0, 0; p < len(s); {
+		var r rune
+		r, sz = utf8.DecodeRuneInString(s[p:])
+		out := s[p : p+sz]
+		chars := 1
+		if !unicode.IsPrint(r) || r == utf8.RuneError || r == '"' {
+			switch sz {
+			case 1:
+				out = fmt.Sprintf("\\x%02x", s[p])
+			case 2, 3:
+				out = fmt.Sprintf("\\u%04x", r)
+			case 4:
+				out = fmt.Sprintf("\\U%08x", r)
+			}
+			chars = len(out)
+		}
+		if n -= chars; n < 0 {
+			nLines++
+			if explicitParens && nLines&63 == 63 {
+				w.printf("\") + (\"")
+			}
+			w.printf("\" +\n\"")
+			n = max - len(out)
+		}
+		w.printf("%s", out)
+		p += sz
+	}
+	w.printf(`"`)
+	if explicitParens {
+		w.printf(`)`)
+	}
+}
+
+// WriteSlice writes a slice value.
+func (w *CodeWriter) WriteSlice(x interface{}) {
+	w.writeSlice(x, false)
+}
+
+// WriteArray writes an array value.
+func (w *CodeWriter) WriteArray(x interface{}) {
+	w.writeSlice(x, true)
+}
+
+func (w *CodeWriter) writeSlice(x interface{}, isArray bool) {
+	v := reflect.ValueOf(x)
+	w.gob.Encode(v.Len())
+	w.Size += v.Len() * int(v.Type().Elem().Size())
+	name := typeName(x)
+	if isArray {
+		name = fmt.Sprintf("[%d]%s", v.Len(), name[strings.Index(name, "]")+1:])
+	}
+	if isArray {
+		w.printf("%s{\n", name)
+	} else {
+		w.printf("%s{ // %d elements\n", name, v.Len())
+	}
+
+	switch kind := v.Type().Elem().Kind(); kind {
+	case reflect.String:
+		for _, s := range x.([]string) {
+			w.WriteString(s)
+			w.printf(",\n")
+		}
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		// nLine and nBlock are the number of elements per line and block.
+		nLine, nBlock, format := 8, 64, "%d,"
+		switch kind {
+		case reflect.Uint8:
+			format = "%#02x,"
+		case reflect.Uint16:
+			format = "%#04x,"
+		case reflect.Uint32:
+			nLine, nBlock, format = 4, 32, "%#08x,"
+		case reflect.Uint, reflect.Uint64:
+			nLine, nBlock, format = 4, 32, "%#016x,"
+		case reflect.Int8:
+			nLine = 16
+		}
+		n := nLine
+		for i := 0; i < v.Len(); i++ {
+			if i%nBlock == 0 && v.Len() > nBlock {
+				w.printf("// Entry %X - %X\n", i, i+nBlock-1)
+			}
+			x := v.Index(i).Interface()
+			w.gob.Encode(x)
+			w.printf(format, x)
+			if n--; n == 0 {
+				n = nLine
+				w.printf("\n")
+			}
+		}
+		w.printf("\n")
+	case reflect.Struct:
+		zero := reflect.Zero(v.Type().Elem()).Interface()
+		for i := 0; i < v.Len(); i++ {
+			x := v.Index(i).Interface()
+			w.gob.EncodeValue(v)
+			if !reflect.DeepEqual(zero, x) {
+				line := fmt.Sprintf("%#v,\n", x)
+				line = line[strings.IndexByte(line, '{'):]
+				w.printf("%d: ", i)
+				w.printf(line)
+			}
+		}
+	case reflect.Array:
+		for i := 0; i < v.Len(); i++ {
+			w.printf("%d: %#v,\n", i, v.Index(i).Interface())
+		}
+	default:
+		panic("gen: slice elem type not supported")
+	}
+	w.printf("}")
+}
+
+// WriteType writes a definition of the type of the given value and returns the
+// type name.
+func (w *CodeWriter) WriteType(x interface{}) string {
+	t := reflect.TypeOf(x)
+	w.printf("type %s struct {\n", t.Name())
+	for i := 0; i < t.NumField(); i++ {
+		w.printf("\t%s %s\n", t.Field(i).Name, t.Field(i).Type)
+	}
+	w.printf("}\n")
+	return t.Name()
+}
+
+// typeName returns the name of the go type of x.
+func typeName(x interface{}) string {
+	t := reflect.ValueOf(x).Type()
+	return strings.Replace(fmt.Sprint(t), "main.", "", 1)
+}

vendor/golang.org/x/text/internal/gen/gen.go

@@ -0,0 +1,281 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package gen contains common code for the various code generation tools in the
+// text repository. Its usage ensures consistency between tools.
+//
+// This package defines command line flags that are common to most generation
+// tools. The flags allow for specifying specific Unicode and CLDR versions
+// in the public Unicode data repository (http://www.unicode.org/Public).
+//
+// A local Unicode data mirror can be set through the flag -local or the
+// environment variable UNICODE_DIR. The former takes precedence. The local
+// directory should follow the same structure as the public repository.
+//
+// IANA data can also optionally be mirrored by putting it in the iana directory
+// rooted at the top of the local mirror. Beware, though, that IANA data is not
+// versioned. So it is up to the developer to use the right version.
+package gen // import "golang.org/x/text/internal/gen"
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/build"
+	"go/format"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+	"unicode"
+
+	"golang.org/x/text/unicode/cldr"
+)
+
+var (
+	url = flag.String("url",
+		"http://www.unicode.org/Public",
+		"URL of Unicode database directory")
+	iana = flag.String("iana",
+		"http://www.iana.org",
+		"URL of the IANA repository")
+	unicodeVersion = flag.String("unicode",
+		getEnv("UNICODE_VERSION", unicode.Version),
+		"unicode version to use")
+	cldrVersion = flag.String("cldr",
+		getEnv("CLDR_VERSION", cldr.Version),
+		"cldr version to use")
+)
+
+func getEnv(name, def string) string {
+	if v := os.Getenv(name); v != "" {
+		return v
+	}
+	return def
+}
+
+// Init performs common initialization for a gen command. It parses the flags
+// and sets up the standard logging parameters.
+func Init() {
+	log.SetPrefix("")
+	log.SetFlags(log.Lshortfile)
+	flag.Parse()
+}
+
+const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package %s
+
+`
+
+// UnicodeVersion reports the requested Unicode version.
+func UnicodeVersion() string {
+	return *unicodeVersion
+}
+
+// UnicodeVersion reports the requested CLDR version.
+func CLDRVersion() string {
+	return *cldrVersion
+}
+
+// IsLocal reports whether data files are available locally.
+func IsLocal() bool {
+	dir, err := localReadmeFile()
+	if err != nil {
+		return false
+	}
+	if _, err = os.Stat(dir); err != nil {
+		return false
+	}
+	return true
+}
+
+// OpenUCDFile opens the requested UCD file. The file is specified relative to
+// the public Unicode root directory. It will call log.Fatal if there are any
+// errors.
+func OpenUCDFile(file string) io.ReadCloser {
+	return openUnicode(path.Join(*unicodeVersion, "ucd", file))
+}
+
+// OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there
+// are any errors.
+func OpenCLDRCoreZip() io.ReadCloser {
+	return OpenUnicodeFile("cldr", *cldrVersion, "core.zip")
+}
+
+// OpenUnicodeFile opens the requested file of the requested category from the
+// root of the Unicode data archive. The file is specified relative to the
+// public Unicode root directory. If version is "", it will use the default
+// Unicode version. It will call log.Fatal if there are any errors.
+func OpenUnicodeFile(category, version, file string) io.ReadCloser {
+	if version == "" {
+		version = UnicodeVersion()
+	}
+	return openUnicode(path.Join(category, version, file))
+}
+
+// OpenIANAFile opens the requested IANA file. The file is specified relative
+// to the IANA root, which is typically either http://www.iana.org or the
+// iana directory in the local mirror. It will call log.Fatal if there are any
+// errors.
+func OpenIANAFile(path string) io.ReadCloser {
+	return Open(*iana, "iana", path)
+}
+
+var (
+	dirMutex sync.Mutex
+	localDir string
+)
+
+const permissions = 0755
+
+func localReadmeFile() (string, error) {
+	p, err := build.Import("golang.org/x/text", "", build.FindOnly)
+	if err != nil {
+		return "", fmt.Errorf("Could not locate package: %v", err)
+	}
+	return filepath.Join(p.Dir, "DATA", "README"), nil
+}
+
+func getLocalDir() string {
+	dirMutex.Lock()
+	defer dirMutex.Unlock()
+
+	readme, err := localReadmeFile()
+	if err != nil {
+		log.Fatal(err)
+	}
+	dir := filepath.Dir(readme)
+	if _, err := os.Stat(readme); err != nil {
+		if err := os.MkdirAll(dir, permissions); err != nil {
+			log.Fatalf("Could not create directory: %v", err)
+		}
+		ioutil.WriteFile(readme, []byte(readmeTxt), permissions)
+	}
+	return dir
+}
+
+const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT.
+
+This directory contains downloaded files used to generate the various tables
+in the golang.org/x/text subrepo.
+
+Note that the language subtag repo (iana/assignments/language-subtag-registry)
+and all other times in the iana subdirectory are not versioned and will need
+to be periodically manually updated. The easiest way to do this is to remove
+the entire iana directory. This is mostly of concern when updating the language
+package.
+`
+
+// Open opens subdir/path if a local directory is specified and the file exists,
+// where subdir is a directory relative to the local root, or fetches it from
+// urlRoot/path otherwise. It will call log.Fatal if there are any errors.
+func Open(urlRoot, subdir, path string) io.ReadCloser {
+	file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path))
+	return open(file, urlRoot, path)
+}
+
+func openUnicode(path string) io.ReadCloser {
+	file := filepath.Join(getLocalDir(), filepath.FromSlash(path))
+	return open(file, *url, path)
+}
+
+// TODO: automatically periodically update non-versioned files.
+
+func open(file, urlRoot, path string) io.ReadCloser {
+	if f, err := os.Open(file); err == nil {
+		return f
+	}
+	r := get(urlRoot, path)
+	defer r.Close()
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Fatalf("Could not download file: %v", err)
+	}
+	os.MkdirAll(filepath.Dir(file), permissions)
+	if err := ioutil.WriteFile(file, b, permissions); err != nil {
+		log.Fatalf("Could not create file: %v", err)
+	}
+	return ioutil.NopCloser(bytes.NewReader(b))
+}
+
+func get(root, path string) io.ReadCloser {
+	url := root + "/" + path
+	fmt.Printf("Fetching %s...", url)
+	defer fmt.Println(" done.")
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Fatalf("HTTP GET: %v", err)
+	}
+	if resp.StatusCode != 200 {
+		log.Fatalf("Bad GET status for %q: %q", url, resp.Status)
+	}
+	return resp.Body
+}
+
+// TODO: use Write*Version in all applicable packages.
+
+// WriteUnicodeVersion writes a constant for the Unicode version from which the
+// tables are generated.
+func WriteUnicodeVersion(w io.Writer) {
+	fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n")
+	fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion())
+}
+
+// WriteCLDRVersion writes a constant for the CLDR version from which the
+// tables are generated.
+func WriteCLDRVersion(w io.Writer) {
+	fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n")
+	fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion())
+}
+
+// WriteGoFile prepends a standard file comment and package statement to the
+// given bytes, applies gofmt, and writes them to a file with the given name.
+// It will call log.Fatal if there are any errors.
+func WriteGoFile(filename, pkg string, b []byte) {
+	w, err := os.Create(filename)
+	if err != nil {
+		log.Fatalf("Could not create file %s: %v", filename, err)
+	}
+	defer w.Close()
+	if _, err = WriteGo(w, pkg, b); err != nil {
+		log.Fatalf("Error writing file %s: %v", filename, err)
+	}
+}
+
+// WriteGo prepends a standard file comment and package statement to the given
+// bytes, applies gofmt, and writes them to w.
+func WriteGo(w io.Writer, pkg string, b []byte) (n int, err error) {
+	src := []byte(fmt.Sprintf(header, pkg))
+	src = append(src, b...)
+	formatted, err := format.Source(src)
+	if err != nil {
+		// Print the generated code even in case of an error so that the
+		// returned error can be meaningfully interpreted.
+		n, _ = w.Write(src)
+		return n, err
+	}
+	return w.Write(formatted)
+}
+
+// Repackage rewrites a Go file from belonging to package main to belonging to
+// the given package.
+func Repackage(inFile, outFile, pkg string) {
+	src, err := ioutil.ReadFile(inFile)
+	if err != nil {
+		log.Fatalf("reading %s: %v", inFile, err)
+	}
+	const toDelete = "package main\n\n"
+	i := bytes.Index(src, []byte(toDelete))
+	if i < 0 {
+		log.Fatalf("Could not find %q in %s.", toDelete, inFile)
+	}
+	w := &bytes.Buffer{}
+	w.Write(src[i+len(toDelete):])
+	WriteGoFile(outFile, pkg, w.Bytes())
+}

vendor/golang.org/x/text/internal/triegen/compact.go

@@ -0,0 +1,58 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package triegen
+
+// This file defines Compacter and its implementations.
+
+import "io"
+
+// A Compacter generates an alternative, more space-efficient way to store a
+// trie value block. A trie value block holds all possible values for the last
+// byte of a UTF-8 encoded rune. Excluding ASCII characters, a trie value block
+// always has 64 values, as a UTF-8 encoding ends with a byte in [0x80, 0xC0).
+type Compacter interface {
+	// Size returns whether the Compacter could encode the given block as well
+	// as its size in case it can. len(v) is always 64.
+	Size(v []uint64) (sz int, ok bool)
+
+	// Store stores the block using the Compacter's compression method.
+	// It returns a handle with which the block can be retrieved.
+	// len(v) is always 64.
+	Store(v []uint64) uint32
+
+	// Print writes the data structures associated to the given store to w.
+	Print(w io.Writer) error
+
+	// Handler returns the name of a function that gets called during trie
+	// lookup for blocks generated by the Compacter. The function should be of
+	// the form func (n uint32, b byte) uint64, where n is the index returned by
+	// the Compacter's Store method and b is the last byte of the UTF-8
+	// encoding, where 0x80 <= b < 0xC0, for which to do the lookup in the
+	// block.
+	Handler() string
+}
+
+// simpleCompacter is the default Compacter used by builder. It implements a
+// normal trie block.
+type simpleCompacter builder
+
+func (b *simpleCompacter) Size([]uint64) (sz int, ok bool) {
+	return blockSize * b.ValueSize, true
+}
+
+func (b *simpleCompacter) Store(v []uint64) uint32 {
+	h := uint32(len(b.ValueBlocks) - blockOffset)
+	b.ValueBlocks = append(b.ValueBlocks, v)
+	return h
+}
+
+func (b *simpleCompacter) Print(io.Writer) error {
+	// Structures are printed in print.go.
+	return nil
+}
+
+func (b *simpleCompacter) Handler() string {
+	panic("Handler should be special-cased for this Compacter")
+}

vendor/golang.org/x/text/internal/triegen/print.go

@@ -0,0 +1,251 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package triegen
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"text/template"
+)
+
+// print writes all the data structures as well as the code necessary to use the
+// trie to w.
+func (b *builder) print(w io.Writer) error {
+	b.Stats.NValueEntries = len(b.ValueBlocks) * blockSize
+	b.Stats.NValueBytes = len(b.ValueBlocks) * blockSize * b.ValueSize
+	b.Stats.NIndexEntries = len(b.IndexBlocks) * blockSize
+	b.Stats.NIndexBytes = len(b.IndexBlocks) * blockSize * b.IndexSize
+	b.Stats.NHandleBytes = len(b.Trie) * 2 * b.IndexSize
+
+	// If we only have one root trie, all starter blocks are at position 0 and
+	// we can access the arrays directly.
+	if len(b.Trie) == 1 {
+		// At this point we cannot refer to the generated tables directly.
+		b.ASCIIBlock = b.Name + "Values"
+		b.StarterBlock = b.Name + "Index"
+	} else {
+		// Otherwise we need to have explicit starter indexes in the trie
+		// structure.
+		b.ASCIIBlock = "t.ascii"
+		b.StarterBlock = "t.utf8Start"
+	}
+
+	b.SourceType = "[]byte"
+	if err := lookupGen.Execute(w, b); err != nil {
+		return err
+	}
+
+	b.SourceType = "string"
+	if err := lookupGen.Execute(w, b); err != nil {
+		return err
+	}
+
+	if err := trieGen.Execute(w, b); err != nil {
+		return err
+	}
+
+	for _, c := range b.Compactions {
+		if err := c.c.Print(w); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func printValues(n int, values []uint64) string {
+	w := &bytes.Buffer{}
+	boff := n * blockSize
+	fmt.Fprintf(w, "\t// Block %#x, offset %#x", n, boff)
+	var newline bool
+	for i, v := range values {
+		if i%6 == 0 {
+			newline = true
+		}
+		if v != 0 {
+			if newline {
+				fmt.Fprintf(w, "\n")
+				newline = false
+			}
+			fmt.Fprintf(w, "\t%#02x:%#04x, ", boff+i, v)
+		}
+	}
+	return w.String()
+}
+
+func printIndex(b *builder, nr int, n *node) string {
+	w := &bytes.Buffer{}
+	boff := nr * blockSize
+	fmt.Fprintf(w, "\t// Block %#x, offset %#x", nr, boff)
+	var newline bool
+	for i, c := range n.children {
+		if i%8 == 0 {
+			newline = true
+		}
+		if c != nil {
+			v := b.Compactions[c.index.compaction].Offset + uint32(c.index.index)
+			if v != 0 {
+				if newline {
+					fmt.Fprintf(w, "\n")
+					newline = false
+				}
+				fmt.Fprintf(w, "\t%#02x:%#02x, ", boff+i, v)
+			}
+		}
+	}
+	return w.String()
+}
+
+var (
+	trieGen = template.Must(template.New("trie").Funcs(template.FuncMap{
+		"printValues": printValues,
+		"printIndex":  printIndex,
+		"title":       strings.Title,
+		"dec":         func(x int) int { return x - 1 },
+		"psize": func(n int) string {
+			return fmt.Sprintf("%d bytes (%.2f KiB)", n, float64(n)/1024)
+		},
+	}).Parse(trieTemplate))
+	lookupGen = template.Must(template.New("lookup").Parse(lookupTemplate))
+)
+
+// TODO: consider the return type of lookup. It could be uint64, even if the
+// internal value type is smaller. We will have to verify this with the
+// performance of unicode/norm, which is very sensitive to such changes.
+const trieTemplate = `{{$b := .}}{{$multi := gt (len .Trie) 1}}
+// {{.Name}}Trie. Total size: {{psize .Size}}. Checksum: {{printf "%08x" .Checksum}}.
+type {{.Name}}Trie struct { {{if $multi}}
+	ascii []{{.ValueType}} // index for ASCII bytes
+	utf8Start  []{{.IndexType}} // index for UTF-8 bytes >= 0xC0
+{{end}}}
+
+func new{{title .Name}}Trie(i int) *{{.Name}}Trie { {{if $multi}}
+	h := {{.Name}}TrieHandles[i]
+	return &{{.Name}}Trie{ {{.Name}}Values[uint32(h.ascii)<<6:], {{.Name}}Index[uint32(h.multi)<<6:] }
+}
+
+type {{.Name}}TrieHandle struct {
+	ascii, multi {{.IndexType}}
+}
+
+// {{.Name}}TrieHandles: {{len .Trie}} handles, {{.Stats.NHandleBytes}} bytes
+var {{.Name}}TrieHandles = [{{len .Trie}}]{{.Name}}TrieHandle{
+{{range .Trie}}	{ {{.ASCIIIndex}}, {{.StarterIndex}} }, // {{printf "%08x" .Checksum}}: {{.Name}}
+{{end}}}{{else}}
+	return &{{.Name}}Trie{}
+}
+{{end}}
+// lookupValue determines the type of block n and looks up the value for b.
+func (t *{{.Name}}Trie) lookupValue(n uint32, b byte) {{.ValueType}}{{$last := dec (len .Compactions)}} {
+	switch { {{range $i, $c := .Compactions}}
+		{{if eq $i $last}}default{{else}}case n < {{$c.Cutoff}}{{end}}:{{if ne $i 0}}
+			n -= {{$c.Offset}}{{end}}
+			return {{print $b.ValueType}}({{$c.Handler}}){{end}}
+	}
+}
+
+// {{.Name}}Values: {{len .ValueBlocks}} blocks, {{.Stats.NValueEntries}} entries, {{.Stats.NValueBytes}} bytes
+// The third block is the zero block.
+var {{.Name}}Values = [{{.Stats.NValueEntries}}]{{.ValueType}} {
+{{range $i, $v := .ValueBlocks}}{{printValues $i $v}}
+{{end}}}
+
+// {{.Name}}Index: {{len .IndexBlocks}} blocks, {{.Stats.NIndexEntries}} entries, {{.Stats.NIndexBytes}} bytes
+// Block 0 is the zero block.
+var {{.Name}}Index = [{{.Stats.NIndexEntries}}]{{.IndexType}} {
+{{range $i, $v := .IndexBlocks}}{{printIndex $b $i $v}}
+{{end}}}
+`
+
+// TODO: consider allowing zero-length strings after evaluating performance with
+// unicode/norm.
+const lookupTemplate = `
+// lookup{{if eq .SourceType "string"}}String{{end}} returns the trie value for the first UTF-8 encoding in s and
+// the width in bytes of this encoding. The size will be 0 if s does not
+// hold enough bytes to complete the encoding. len(s) must be greater than 0.
+func (t *{{.Name}}Trie) lookup{{if eq .SourceType "string"}}String{{end}}(s {{.SourceType}}) (v {{.ValueType}}, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return {{.ASCIIBlock}}[c0], 1
+	case c0 < 0xC2:
+		return 0, 1  // Illegal UTF-8: not a starter, not ASCII.
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := {{.StarterBlock}}[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return 0, 1 // Illegal UTF-8: not a continuation byte.
+		}
+		return t.lookupValue(uint32(i), c1), 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := {{.StarterBlock}}[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return 0, 1 // Illegal UTF-8: not a continuation byte.
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = {{.Name}}Index[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return 0, 2 // Illegal UTF-8: not a continuation byte.
+		}
+		return t.lookupValue(uint32(i), c2), 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := {{.StarterBlock}}[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return 0, 1 // Illegal UTF-8: not a continuation byte.
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = {{.Name}}Index[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return 0, 2 // Illegal UTF-8: not a continuation byte.
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = {{.Name}}Index[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return 0, 3 // Illegal UTF-8: not a continuation byte.
+		}
+		return t.lookupValue(uint32(i), c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}
+
+// lookup{{if eq .SourceType "string"}}String{{end}}Unsafe returns the trie value for the first UTF-8 encoding in s.
+// s must start with a full and valid UTF-8 encoded rune.
+func (t *{{.Name}}Trie) lookup{{if eq .SourceType "string"}}String{{end}}Unsafe(s {{.SourceType}}) {{.ValueType}} {
+	c0 := s[0]
+	if c0 < 0x80 { // is ASCII
+		return {{.ASCIIBlock}}[c0]
+	}
+	i := {{.StarterBlock}}[c0]
+	if c0 < 0xE0 { // 2-byte UTF-8
+		return t.lookupValue(uint32(i), s[1])
+	}
+	i = {{.Name}}Index[uint32(i)<<6+uint32(s[1])]
+	if c0 < 0xF0 { // 3-byte UTF-8
+		return t.lookupValue(uint32(i), s[2])
+	}
+	i = {{.Name}}Index[uint32(i)<<6+uint32(s[2])]
+	if c0 < 0xF8 { // 4-byte UTF-8
+		return t.lookupValue(uint32(i), s[3])
+	}
+	return 0
+}
+`

vendor/golang.org/x/text/internal/triegen/triegen.go

@@ -0,0 +1,494 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package triegen implements a code generator for a trie for associating
+// unsigned integer values with UTF-8 encoded runes.
+//
+// Many of the go.text packages use tries for storing per-rune information.  A
+// trie is especially useful if many of the runes have the same value. If this
+// is the case, many blocks can be expected to be shared allowing for
+// information on many runes to be stored in little space.
+//
+// As most of the lookups are done directly on []byte slices, the tries use the
+// UTF-8 bytes directly for the lookup. This saves a conversion from UTF-8 to
+// runes and contributes a little bit to better performance. It also naturally
+// provides a fast path for ASCII.
+//
+// Space is also an issue. There are many code points defined in Unicode and as
+// a result tables can get quite large. So every byte counts. The triegen
+// package automatically chooses the smallest integer values to represent the
+// tables. Compacters allow further compression of the trie by allowing for
+// alternative representations of individual trie blocks.
+//
+// triegen allows generating multiple tries as a single structure. This is
+// useful when, for example, one wants to generate tries for several languages
+// that have a lot of values in common. Some existing libraries for
+// internationalization store all per-language data as a dynamically loadable
+// chunk. The go.text packages are designed with the assumption that the user
+// typically wants to compile in support for all supported languages, in line
+// with the approach common to Go to create a single standalone binary. The
+// multi-root trie approach can give significant storage savings in this
+// scenario.
+//
+// triegen generates both tables and code. The code is optimized to use the
+// automatically chosen data types. The following code is generated for a Trie
+// or multiple Tries named "foo":
+//	- type fooTrie
+//		The trie type.
+//
+//	- func newFooTrie(x int) *fooTrie
+//		Trie constructor, where x is the index of the trie passed to Gen.
+//
+//	- func (t *fooTrie) lookup(s []byte) (v uintX, sz int)
+//		The lookup method, where uintX is automatically chosen.
+//
+//	- func lookupString, lookupUnsafe and lookupStringUnsafe
+//		Variants of the above.
+//
+//	- var fooValues and fooIndex and any tables generated by Compacters.
+//		The core trie data.
+//
+//	- var fooTrieHandles
+//		Indexes of starter blocks in case of multiple trie roots.
+//
+// It is recommended that users test the generated trie by checking the returned
+// value for every rune. Such exhaustive tests are possible as the the number of
+// runes in Unicode is limited.
+package triegen // import "golang.org/x/text/internal/triegen"
+
+// TODO: Arguably, the internally optimized data types would not have to be
+// exposed in the generated API. We could also investigate not generating the
+// code, but using it through a package. We would have to investigate the impact
+// on performance of making such change, though. For packages like unicode/norm,
+// small changes like this could tank performance.
+
+import (
+	"encoding/binary"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"log"
+	"unicode/utf8"
+)
+
+// builder builds a set of tries for associating values with runes. The set of
+// tries can share common index and value blocks.
+type builder struct {
+	Name string
+
+	// ValueType is the type of the trie values looked up.
+	ValueType string
+
+	// ValueSize is the byte size of the ValueType.
+	ValueSize int
+
+	// IndexType is the type of trie index values used for all UTF-8 bytes of
+	// a rune except the last one.
+	IndexType string
+
+	// IndexSize is the byte size of the IndexType.
+	IndexSize int
+
+	// SourceType is used when generating the lookup functions. If the user
+	// requests StringSupport, all lookup functions will be generated for
+	// string input as well.
+	SourceType string
+
+	Trie []*Trie
+
+	IndexBlocks []*node
+	ValueBlocks [][]uint64
+	Compactions []compaction
+	Checksum    uint64
+
+	ASCIIBlock   string
+	StarterBlock string
+
+	indexBlockIdx map[uint64]int
+	valueBlockIdx map[uint64]nodeIndex
+	asciiBlockIdx map[uint64]int
+
+	// Stats are used to fill out the template.
+	Stats struct {
+		NValueEntries int
+		NValueBytes   int
+		NIndexEntries int
+		NIndexBytes   int
+		NHandleBytes  int
+	}
+
+	err error
+}
+
+// A nodeIndex encodes the index of a node, which is defined by the compaction
+// which stores it and an index within the compaction. For internal nodes, the
+// compaction is always 0.
+type nodeIndex struct {
+	compaction int
+	index      int
+}
+
+// compaction keeps track of stats used for the compaction.
+type compaction struct {
+	c         Compacter
+	blocks    []*node
+	maxHandle uint32
+	totalSize int
+
+	// Used by template-based generator and thus exported.
+	Cutoff  uint32
+	Offset  uint32
+	Handler string
+}
+
+func (b *builder) setError(err error) {
+	if b.err == nil {
+		b.err = err
+	}
+}
+
+// An Option can be passed to Gen.
+type Option func(b *builder) error
+
+// Compact configures the trie generator to use the given Compacter.
+func Compact(c Compacter) Option {
+	return func(b *builder) error {
+		b.Compactions = append(b.Compactions, compaction{
+			c:       c,
+			Handler: c.Handler() + "(n, b)"})
+		return nil
+	}
+}
+
+// Gen writes Go code for a shared trie lookup structure to w for the given
+// Tries. The generated trie type will be called nameTrie. newNameTrie(x) will
+// return the *nameTrie for tries[x]. A value can be looked up by using one of
+// the various lookup methods defined on nameTrie. It returns the table size of
+// the generated trie.
+func Gen(w io.Writer, name string, tries []*Trie, opts ...Option) (sz int, err error) {
+	// The index contains two dummy blocks, followed by the zero block. The zero
+	// block is at offset 0x80, so that the offset for the zero block for
+	// continuation bytes is 0.
+	b := &builder{
+		Name:        name,
+		Trie:        tries,
+		IndexBlocks: []*node{{}, {}, {}},
+		Compactions: []compaction{{
+			Handler: name + "Values[n<<6+uint32(b)]",
+		}},
+		// The 0 key in indexBlockIdx and valueBlockIdx is the hash of the zero
+		// block.
+		indexBlockIdx: map[uint64]int{0: 0},
+		valueBlockIdx: map[uint64]nodeIndex{0: {}},
+		asciiBlockIdx: map[uint64]int{},
+	}
+	b.Compactions[0].c = (*simpleCompacter)(b)
+
+	for _, f := range opts {
+		if err := f(b); err != nil {
+			return 0, err
+		}
+	}
+	b.build()
+	if b.err != nil {
+		return 0, b.err
+	}
+	if err = b.print(w); err != nil {
+		return 0, err
+	}
+	return b.Size(), nil
+}
+
+// A Trie represents a single root node of a trie. A builder may build several
+// overlapping tries at once.
+type Trie struct {
+	root *node
+
+	hiddenTrie
+}
+
+// hiddenTrie contains values we want to be visible to the template generator,
+// but hidden from the API documentation.
+type hiddenTrie struct {
+	Name         string
+	Checksum     uint64
+	ASCIIIndex   int
+	StarterIndex int
+}
+
+// NewTrie returns a new trie root.
+func NewTrie(name string) *Trie {
+	return &Trie{
+		&node{
+			children: make([]*node, blockSize),
+			values:   make([]uint64, utf8.RuneSelf),
+		},
+		hiddenTrie{Name: name},
+	}
+}
+
+// Gen is a convenience wrapper around the Gen func passing t as the only trie
+// and uses the name passed to NewTrie. It returns the size of the generated
+// tables.
+func (t *Trie) Gen(w io.Writer, opts ...Option) (sz int, err error) {
+	return Gen(w, t.Name, []*Trie{t}, opts...)
+}
+
+// node is a node of the intermediate trie structure.
+type node struct {
+	// children holds this node's children. It is always of length 64.
+	// A child node may be nil.
+	children []*node
+
+	// values contains the values of this node. If it is non-nil, this node is
+	// either a root or leaf node:
+	// For root nodes, len(values) == 128 and it maps the bytes in [0x00, 0x7F].
+	// For leaf nodes, len(values) ==  64 and it maps the bytes in [0x80, 0xBF].
+	values []uint64
+
+	index nodeIndex
+}
+
+// Insert associates value with the given rune. Insert will panic if a non-zero
+// value is passed for an invalid rune.
+func (t *Trie) Insert(r rune, value uint64) {
+	if value == 0 {
+		return
+	}
+	s := string(r)
+	if []rune(s)[0] != r && value != 0 {
+		// Note: The UCD tables will always assign what amounts to a zero value
+		// to a surrogate. Allowing a zero value for an illegal rune allows
+		// users to iterate over [0..MaxRune] without having to explicitly
+		// exclude surrogates, which would be tedious.
+		panic(fmt.Sprintf("triegen: non-zero value for invalid rune %U", r))
+	}
+	if len(s) == 1 {
+		// It is a root node value (ASCII).
+		t.root.values[s[0]] = value
+		return
+	}
+
+	n := t.root
+	for ; len(s) > 1; s = s[1:] {
+		if n.children == nil {
+			n.children = make([]*node, blockSize)
+		}
+		p := s[0] % blockSize
+		c := n.children[p]
+		if c == nil {
+			c = &node{}
+			n.children[p] = c
+		}
+		if len(s) > 2 && c.values != nil {
+			log.Fatalf("triegen: insert(%U): found internal node with values", r)
+		}
+		n = c
+	}
+	if n.values == nil {
+		n.values = make([]uint64, blockSize)
+	}
+	if n.children != nil {
+		log.Fatalf("triegen: insert(%U): found leaf node that also has child nodes", r)
+	}
+	n.values[s[0]-0x80] = value
+}
+
+// Size returns the number of bytes the generated trie will take to store. It
+// needs to be exported as it is used in the templates.
+func (b *builder) Size() int {
+	// Index blocks.
+	sz := len(b.IndexBlocks) * blockSize * b.IndexSize
+
+	// Skip the first compaction, which represents the normal value blocks, as
+	// its totalSize does not account for the ASCII blocks, which are managed
+	// separately.
+	sz += len(b.ValueBlocks) * blockSize * b.ValueSize
+	for _, c := range b.Compactions[1:] {
+		sz += c.totalSize
+	}
+
+	// TODO: this computation does not account for the fixed overhead of a using
+	// a compaction, either code or data. As for data, though, the typical
+	// overhead of data is in the order of bytes (2 bytes for cases). Further,
+	// the savings of using a compaction should anyway be substantial for it to
+	// be worth it.
+
+	// For multi-root tries, we also need to account for the handles.
+	if len(b.Trie) > 1 {
+		sz += 2 * b.IndexSize * len(b.Trie)
+	}
+	return sz
+}
+
+func (b *builder) build() {
+	// Compute the sizes of the values.
+	var vmax uint64
+	for _, t := range b.Trie {
+		vmax = maxValue(t.root, vmax)
+	}
+	b.ValueType, b.ValueSize = getIntType(vmax)
+
+	// Compute all block allocations.
+	// TODO: first compute the ASCII blocks for all tries and then the other
+	// nodes. ASCII blocks are more restricted in placement, as they require two
+	// blocks to be placed consecutively. Processing them first may improve
+	// sharing (at least one zero block can be expected to be saved.)
+	for _, t := range b.Trie {
+		b.Checksum += b.buildTrie(t)
+	}
+
+	// Compute the offsets for all the Compacters.
+	offset := uint32(0)
+	for i := range b.Compactions {
+		c := &b.Compactions[i]
+		c.Offset = offset
+		offset += c.maxHandle + 1
+		c.Cutoff = offset
+	}
+
+	// Compute the sizes of indexes.
+	// TODO: different byte positions could have different sizes. So far we have
+	// not found a case where this is beneficial.
+	imax := uint64(b.Compactions[len(b.Compactions)-1].Cutoff)
+	for _, ib := range b.IndexBlocks {
+		if x := uint64(ib.index.index); x > imax {
+			imax = x
+		}
+	}
+	b.IndexType, b.IndexSize = getIntType(imax)
+}
+
+func maxValue(n *node, max uint64) uint64 {
+	if n == nil {
+		return max
+	}
+	for _, c := range n.children {
+		max = maxValue(c, max)
+	}
+	for _, v := range n.values {
+		if max < v {
+			max = v
+		}
+	}
+	return max
+}
+
+func getIntType(v uint64) (string, int) {
+	switch {
+	case v < 1<<8:
+		return "uint8", 1
+	case v < 1<<16:
+		return "uint16", 2
+	case v < 1<<32:
+		return "uint32", 4
+	}
+	return "uint64", 8
+}
+
+const (
+	blockSize = 64
+
+	// Subtract two blocks to offset 0x80, the first continuation byte.
+	blockOffset = 2
+
+	// Subtract three blocks to offset 0xC0, the first non-ASCII starter.
+	rootBlockOffset = 3
+)
+
+var crcTable = crc64.MakeTable(crc64.ISO)
+
+func (b *builder) buildTrie(t *Trie) uint64 {
+	n := t.root
+
+	// Get the ASCII offset. For the first trie, the ASCII block will be at
+	// position 0.
+	hasher := crc64.New(crcTable)
+	binary.Write(hasher, binary.BigEndian, n.values)
+	hash := hasher.Sum64()
+
+	v, ok := b.asciiBlockIdx[hash]
+	if !ok {
+		v = len(b.ValueBlocks)
+		b.asciiBlockIdx[hash] = v
+
+		b.ValueBlocks = append(b.ValueBlocks, n.values[:blockSize], n.values[blockSize:])
+		if v == 0 {
+			// Add the zero block at position 2 so that it will be assigned a
+			// zero reference in the lookup blocks.
+			// TODO: always do this? This would allow us to remove a check from
+			// the trie lookup, but at the expense of extra space. Analyze
+			// performance for unicode/norm.
+			b.ValueBlocks = append(b.ValueBlocks, make([]uint64, blockSize))
+		}
+	}
+	t.ASCIIIndex = v
+
+	// Compute remaining offsets.
+	t.Checksum = b.computeOffsets(n, true)
+	// We already subtracted the normal blockOffset from the index. Subtract the
+	// difference for starter bytes.
+	t.StarterIndex = n.index.index - (rootBlockOffset - blockOffset)
+	return t.Checksum
+}
+
+func (b *builder) computeOffsets(n *node, root bool) uint64 {
+	// For the first trie, the root lookup block will be at position 3, which is
+	// the offset for UTF-8 non-ASCII starter bytes.
+	first := len(b.IndexBlocks) == rootBlockOffset
+	if first {
+		b.IndexBlocks = append(b.IndexBlocks, n)
+	}
+
+	// We special-case the cases where all values recursively are 0. This allows
+	// for the use of a zero block to which all such values can be directed.
+	hash := uint64(0)
+	if n.children != nil || n.values != nil {
+		hasher := crc64.New(crcTable)
+		for _, c := range n.children {
+			var v uint64
+			if c != nil {
+				v = b.computeOffsets(c, false)
+			}
+			binary.Write(hasher, binary.BigEndian, v)
+		}
+		binary.Write(hasher, binary.BigEndian, n.values)
+		hash = hasher.Sum64()
+	}
+
+	if first {
+		b.indexBlockIdx[hash] = rootBlockOffset - blockOffset
+	}
+
+	// Compacters don't apply to internal nodes.
+	if n.children != nil {
+		v, ok := b.indexBlockIdx[hash]
+		if !ok {
+			v = len(b.IndexBlocks) - blockOffset
+			b.IndexBlocks = append(b.IndexBlocks, n)
+			b.indexBlockIdx[hash] = v
+		}
+		n.index = nodeIndex{0, v}
+	} else {
+		h, ok := b.valueBlockIdx[hash]
+		if !ok {
+			bestI, bestSize := 0, blockSize*b.ValueSize
+			for i, c := range b.Compactions[1:] {
+				if sz, ok := c.c.Size(n.values); ok && bestSize > sz {
+					bestI, bestSize = i+1, sz
+				}
+			}
+			c := &b.Compactions[bestI]
+			c.totalSize += bestSize
+			v := c.c.Store(n.values)
+			if c.maxHandle < v {
+				c.maxHandle = v
+			}
+			h = nodeIndex{bestI, int(v)}
+			b.valueBlockIdx[hash] = h
+		}
+		n.index = h
+	}
+	return hash
+}

vendor/golang.org/x/text/internal/ucd/ucd.go

@@ -0,0 +1,376 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package ucd provides a parser for Unicode Character Database files, the
+// format of which is defined in http://www.unicode.org/reports/tr44/. See
+// http://www.unicode.org/Public/UCD/latest/ucd/ for example files.
+//
+// It currently does not support substitutions of missing fields.
+package ucd // import "golang.org/x/text/internal/ucd"
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"log"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// UnicodeData.txt fields.
+const (
+	CodePoint = iota
+	Name
+	GeneralCategory
+	CanonicalCombiningClass
+	BidiClass
+	DecompMapping
+	DecimalValue
+	DigitValue
+	NumericValue
+	BidiMirrored
+	Unicode1Name
+	ISOComment
+	SimpleUppercaseMapping
+	SimpleLowercaseMapping
+	SimpleTitlecaseMapping
+)
+
+// Parse calls f for each entry in the given reader of a UCD file. It will close
+// the reader upon return. It will call log.Fatal if any error occurred.
+//
+// This implements the most common usage pattern of using Parser.
+func Parse(r io.ReadCloser, f func(p *Parser)) {
+	defer r.Close()
+
+	p := New(r)
+	for p.Next() {
+		f(p)
+	}
+	if err := p.Err(); err != nil {
+		r.Close() // os.Exit will cause defers not to be called.
+		log.Fatal(err)
+	}
+}
+
+// An Option is used to configure a Parser.
+type Option func(p *Parser)
+
+func keepRanges(p *Parser) {
+	p.keepRanges = true
+}
+
+var (
+	// KeepRanges prevents the expansion of ranges. The raw ranges can be
+	// obtained by calling Range(0) on the parser.
+	KeepRanges Option = keepRanges
+)
+
+// The Part option register a handler for lines starting with a '@'. The text
+// after a '@' is available as the first field. Comments are handled as usual.
+func Part(f func(p *Parser)) Option {
+	return func(p *Parser) {
+		p.partHandler = f
+	}
+}
+
+// The CommentHandler option passes comments that are on a line by itself to
+// a given handler.
+func CommentHandler(f func(s string)) Option {
+	return func(p *Parser) {
+		p.commentHandler = f
+	}
+}
+
+// A Parser parses Unicode Character Database (UCD) files.
+type Parser struct {
+	scanner *bufio.Scanner
+
+	keepRanges bool // Don't expand rune ranges in field 0.
+
+	err     error
+	comment []byte
+	field   [][]byte
+	// parsedRange is needed in case Range(0) is called more than once for one
+	// field. In some cases this requires scanning ahead.
+	parsedRange          bool
+	rangeStart, rangeEnd rune
+
+	partHandler    func(p *Parser)
+	commentHandler func(s string)
+}
+
+func (p *Parser) setError(err error) {
+	if p.err == nil {
+		p.err = err
+	}
+}
+
+func (p *Parser) getField(i int) []byte {
+	if i >= len(p.field) {
+		return nil
+	}
+	return p.field[i]
+}
+
+// Err returns a non-nil error if any error occurred during parsing.
+func (p *Parser) Err() error {
+	return p.err
+}
+
+// New returns a Parser for the given Reader.
+func New(r io.Reader, o ...Option) *Parser {
+	p := &Parser{
+		scanner: bufio.NewScanner(r),
+	}
+	for _, f := range o {
+		f(p)
+	}
+	return p
+}
+
+// Next parses the next line in the file. It returns true if a line was parsed
+// and false if it reached the end of the file.
+func (p *Parser) Next() bool {
+	if !p.keepRanges && p.rangeStart < p.rangeEnd {
+		p.rangeStart++
+		return true
+	}
+	p.comment = nil
+	p.field = p.field[:0]
+	p.parsedRange = false
+
+	for p.scanner.Scan() {
+		b := p.scanner.Bytes()
+		if len(b) == 0 {
+			continue
+		}
+		if b[0] == '#' {
+			if p.commentHandler != nil {
+				p.commentHandler(strings.TrimSpace(string(b[1:])))
+			}
+			continue
+		}
+
+		// Parse line
+		if i := bytes.IndexByte(b, '#'); i != -1 {
+			p.comment = bytes.TrimSpace(b[i+1:])
+			b = b[:i]
+		}
+		if b[0] == '@' {
+			if p.partHandler != nil {
+				p.field = append(p.field, bytes.TrimSpace(b[1:]))
+				p.partHandler(p)
+				p.field = p.field[:0]
+			}
+			p.comment = nil
+			continue
+		}
+		for {
+			i := bytes.IndexByte(b, ';')
+			if i == -1 {
+				p.field = append(p.field, bytes.TrimSpace(b))
+				break
+			}
+			p.field = append(p.field, bytes.TrimSpace(b[:i]))
+			b = b[i+1:]
+		}
+		if !p.keepRanges {
+			p.rangeStart, p.rangeEnd = p.getRange(0)
+		}
+		return true
+	}
+	p.setError(p.scanner.Err())
+	return false
+}
+
+func parseRune(b []byte) (rune, error) {
+	if len(b) > 2 && b[0] == 'U' && b[1] == '+' {
+		b = b[2:]
+	}
+	x, err := strconv.ParseUint(string(b), 16, 32)
+	return rune(x), err
+}
+
+func (p *Parser) parseRune(b []byte) rune {
+	x, err := parseRune(b)
+	p.setError(err)
+	return x
+}
+
+// Rune parses and returns field i as a rune.
+func (p *Parser) Rune(i int) rune {
+	if i > 0 || p.keepRanges {
+		return p.parseRune(p.getField(i))
+	}
+	return p.rangeStart
+}
+
+// Runes interprets and returns field i as a sequence of runes.
+func (p *Parser) Runes(i int) (runes []rune) {
+	add := func(b []byte) {
+		if b = bytes.TrimSpace(b); len(b) > 0 {
+			runes = append(runes, p.parseRune(b))
+		}
+	}
+	for b := p.getField(i); ; {
+		i := bytes.IndexByte(b, ' ')
+		if i == -1 {
+			add(b)
+			break
+		}
+		add(b[:i])
+		b = b[i+1:]
+	}
+	return
+}
+
+var (
+	errIncorrectLegacyRange = errors.New("ucd: unmatched <* First>")
+
+	// reRange matches one line of a legacy rune range.
+	reRange = regexp.MustCompile("^([0-9A-F]*);<([^,]*), ([^>]*)>(.*)$")
+)
+
+// Range parses and returns field i as a rune range. A range is inclusive at
+// both ends. If the field only has one rune, first and last will be identical.
+// It supports the legacy format for ranges used in UnicodeData.txt.
+func (p *Parser) Range(i int) (first, last rune) {
+	if !p.keepRanges {
+		return p.rangeStart, p.rangeStart
+	}
+	return p.getRange(i)
+}
+
+func (p *Parser) getRange(i int) (first, last rune) {
+	b := p.getField(i)
+	if k := bytes.Index(b, []byte("..")); k != -1 {
+		return p.parseRune(b[:k]), p.parseRune(b[k+2:])
+	}
+	// The first field may not be a rune, in which case we may ignore any error
+	// and set the range as 0..0.
+	x, err := parseRune(b)
+	if err != nil {
+		// Disable range parsing henceforth. This ensures that an error will be
+		// returned if the user subsequently will try to parse this field as
+		// a Rune.
+		p.keepRanges = true
+	}
+	// Special case for UnicodeData that was retained for backwards compatibility.
+	if i == 0 && len(p.field) > 1 && bytes.HasSuffix(p.field[1], []byte("First>")) {
+		if p.parsedRange {
+			return p.rangeStart, p.rangeEnd
+		}
+		mf := reRange.FindStringSubmatch(p.scanner.Text())
+		if mf == nil || !p.scanner.Scan() {
+			p.setError(errIncorrectLegacyRange)
+			return x, x
+		}
+		// Using Bytes would be more efficient here, but Text is a lot easier
+		// and this is not a frequent case.
+		ml := reRange.FindStringSubmatch(p.scanner.Text())
+		if ml == nil || mf[2] != ml[2] || ml[3] != "Last" || mf[4] != ml[4] {
+			p.setError(errIncorrectLegacyRange)
+			return x, x
+		}
+		p.rangeStart, p.rangeEnd = x, p.parseRune(p.scanner.Bytes()[:len(ml[1])])
+		p.parsedRange = true
+		return p.rangeStart, p.rangeEnd
+	}
+	return x, x
+}
+
+// bools recognizes all valid UCD boolean values.
+var bools = map[string]bool{
+	"":      false,
+	"N":     false,
+	"No":    false,
+	"F":     false,
+	"False": false,
+	"Y":     true,
+	"Yes":   true,
+	"T":     true,
+	"True":  true,
+}
+
+// Bool parses and returns field i as a boolean value.
+func (p *Parser) Bool(i int) bool {
+	b := p.getField(i)
+	for s, v := range bools {
+		if bstrEq(b, s) {
+			return v
+		}
+	}
+	p.setError(strconv.ErrSyntax)
+	return false
+}
+
+// Int parses and returns field i as an integer value.
+func (p *Parser) Int(i int) int {
+	x, err := strconv.ParseInt(string(p.getField(i)), 10, 64)
+	p.setError(err)
+	return int(x)
+}
+
+// Uint parses and returns field i as an unsigned integer value.
+func (p *Parser) Uint(i int) uint {
+	x, err := strconv.ParseUint(string(p.getField(i)), 10, 64)
+	p.setError(err)
+	return uint(x)
+}
+
+// Float parses and returns field i as a decimal value.
+func (p *Parser) Float(i int) float64 {
+	x, err := strconv.ParseFloat(string(p.getField(i)), 64)
+	p.setError(err)
+	return x
+}
+
+// String parses and returns field i as a string value.
+func (p *Parser) String(i int) string {
+	return string(p.getField(i))
+}
+
+// Strings parses and returns field i as a space-separated list of strings.
+func (p *Parser) Strings(i int) []string {
+	ss := strings.Split(string(p.getField(i)), " ")
+	for i, s := range ss {
+		ss[i] = strings.TrimSpace(s)
+	}
+	return ss
+}
+
+// Comment returns the comments for the current line.
+func (p *Parser) Comment() string {
+	return string(p.comment)
+}
+
+var errUndefinedEnum = errors.New("ucd: undefined enum value")
+
+// Enum interprets and returns field i as a value that must be one of the values
+// in enum.
+func (p *Parser) Enum(i int, enum ...string) string {
+	b := p.getField(i)
+	for _, s := range enum {
+		if bstrEq(b, s) {
+			return s
+		}
+	}
+	p.setError(errUndefinedEnum)
+	return ""
+}
+
+func bstrEq(b []byte, s string) bool {
+	if len(b) != len(s) {
+		return false
+	}
+	for i, c := range b {
+		if c != s[i] {
+			return false
+		}
+	}
+	return true
+}

vendor/golang.org/x/text/secure/bidirule/bidirule.go

@@ -0,0 +1,342 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bidirule implements the Bidi Rule defined by RFC 5893.
+//
+// This package is under development. The API may change without notice and
+// without preserving backward compatibility.
+package bidirule
+
+import (
+	"errors"
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/bidi"
+)
+
+// This file contains an implementation of RFC 5893: Right-to-Left Scripts for
+// Internationalized Domain Names for Applications (IDNA)
+//
+// A label is an individual component of a domain name.  Labels are usually
+// shown separated by dots; for example, the domain name "www.example.com" is
+// composed of three labels: "www", "example", and "com".
+//
+// An RTL label is a label that contains at least one character of class R, AL,
+// or AN. An LTR label is any label that is not an RTL label.
+//
+// A "Bidi domain name" is a domain name that contains at least one RTL label.
+//
+//  The following guarantees can be made based on the above:
+//
+//  o  In a domain name consisting of only labels that satisfy the rule,
+//     the requirements of Section 3 are satisfied.  Note that even LTR
+//     labels and pure ASCII labels have to be tested.
+//
+//  o  In a domain name consisting of only LDH labels (as defined in the
+//     Definitions document [RFC5890]) and labels that satisfy the rule,
+//     the requirements of Section 3 are satisfied as long as a label
+//     that starts with an ASCII digit does not come after a
+//     right-to-left label.
+//
+//  No guarantee is given for other combinations.
+
+// ErrInvalid indicates a label is invalid according to the Bidi Rule.
+var ErrInvalid = errors.New("bidirule: failed Bidi Rule")
+
+type ruleState uint8
+
+const (
+	ruleInitial ruleState = iota
+	ruleLTR
+	ruleLTRFinal
+	ruleRTL
+	ruleRTLFinal
+	ruleInvalid
+)
+
+type ruleTransition struct {
+	next ruleState
+	mask uint16
+}
+
+var transitions = [...][2]ruleTransition{
+	// [2.1] The first character must be a character with Bidi property L, R, or
+	// AL. If it has the R or AL property, it is an RTL label; if it has the L
+	// property, it is an LTR label.
+	ruleInitial: {
+		{ruleLTRFinal, 1 << bidi.L},
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL},
+	},
+	ruleRTL: {
+		// [2.3] In an RTL label, the end of the label must be a character with
+		// Bidi property R, AL, EN, or AN, followed by zero or more characters
+		// with Bidi property NSM.
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN},
+
+		// [2.2] In an RTL label, only characters with the Bidi properties R,
+		// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.3]
+		{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
+	},
+	ruleRTLFinal: {
+		// [2.3] In an RTL label, the end of the label must be a character with
+		// Bidi property R, AL, EN, or AN, followed by zero or more characters
+		// with Bidi property NSM.
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN | 1<<bidi.NSM},
+
+		// [2.2] In an RTL label, only characters with the Bidi properties R,
+		// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.3] and NSM.
+		{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
+	},
+	ruleLTR: {
+		// [2.6] In an LTR label, the end of the label must be a character with
+		// Bidi property L or EN, followed by zero or more characters with Bidi
+		// property NSM.
+		{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN},
+
+		// [2.5] In an LTR label, only characters with the Bidi properties L,
+		// EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.6].
+		{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
+	},
+	ruleLTRFinal: {
+		// [2.6] In an LTR label, the end of the label must be a character with
+		// Bidi property L or EN, followed by zero or more characters with Bidi
+		// property NSM.
+		{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN | 1<<bidi.NSM},
+
+		// [2.5] In an LTR label, only characters with the Bidi properties L,
+		// EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.6].
+		{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
+	},
+	ruleInvalid: {
+		{ruleInvalid, 0},
+		{ruleInvalid, 0},
+	},
+}
+
+// [2.4] In an RTL label, if an EN is present, no AN may be present, and
+// vice versa.
+const exclusiveRTL = uint16(1<<bidi.EN | 1<<bidi.AN)
+
+// From RFC 5893
+// An RTL label is a label that contains at least one character of type
+// R, AL, or AN.
+//
+// An LTR label is any label that is not an RTL label.
+
+// Direction reports the direction of the given label as defined by RFC 5893.
+// The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func Direction(b []byte) bidi.Direction {
+	for i := 0; i < len(b); {
+		e, sz := bidi.Lookup(b[i:])
+		if sz == 0 {
+			i++
+		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
+	}
+	return bidi.LeftToRight
+}
+
+// DirectionString reports the direction of the given label as defined by RFC
+// 5893. The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func DirectionString(s string) bidi.Direction {
+	for i := 0; i < len(s); {
+		e, sz := bidi.LookupString(s[i:])
+		if sz == 0 {
+			i++
+		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
+	}
+	return bidi.LeftToRight
+}
+
+// Valid reports whether b conforms to the BiDi rule.
+func Valid(b []byte) bool {
+	var t Transformer
+	if n, ok := t.advance(b); !ok || n < len(b) {
+		return false
+	}
+	return t.isFinal()
+}
+
+// ValidString reports whether s conforms to the BiDi rule.
+func ValidString(s string) bool {
+	var t Transformer
+	if n, ok := t.advanceString(s); !ok || n < len(s) {
+		return false
+	}
+	return t.isFinal()
+}
+
+// New returns a Transformer that verifies that input adheres to the Bidi Rule.
+func New() *Transformer {
+	return &Transformer{}
+}
+
+// Transformer implements transform.Transform.
+type Transformer struct {
+	state  ruleState
+	hasRTL bool
+	seen   uint16
+}
+
+// A rule can only be violated for "Bidi Domain names", meaning if one of the
+// following categories has been observed.
+func (t *Transformer) isRTL() bool {
+	const isRTL = 1<<bidi.R | 1<<bidi.AL | 1<<bidi.AN
+	return t.seen&isRTL != 0
+}
+
+func (t *Transformer) isFinal() bool {
+	if !t.isRTL() {
+		return true
+	}
+	return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial
+}
+
+// Reset implements transform.Transformer.
+func (t *Transformer) Reset() { *t = Transformer{} }
+
+// Transform implements transform.Transformer. This Transformer has state and
+// needs to be reset between uses.
+func (t *Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	if len(dst) < len(src) {
+		src = src[:len(dst)]
+		atEOF = false
+		err = transform.ErrShortDst
+	}
+	n, err1 := t.Span(src, atEOF)
+	copy(dst, src[:n])
+	if err == nil || err1 != nil && err1 != transform.ErrShortSrc {
+		err = err1
+	}
+	return n, n, err
+}
+
+// Span returns the first n bytes of src that conform to the Bidi rule.
+func (t *Transformer) Span(src []byte, atEOF bool) (n int, err error) {
+	if t.state == ruleInvalid && t.isRTL() {
+		return 0, ErrInvalid
+	}
+	n, ok := t.advance(src)
+	switch {
+	case !ok:
+		err = ErrInvalid
+	case n < len(src):
+		if !atEOF {
+			err = transform.ErrShortSrc
+			break
+		}
+		err = ErrInvalid
+	case !t.isFinal():
+		err = ErrInvalid
+	}
+	return n, err
+}
+
+// Precomputing the ASCII values decreases running time for the ASCII fast path
+// by about 30%.
+var asciiTable [128]bidi.Properties
+
+func init() {
+	for i := range asciiTable {
+		p, _ := bidi.LookupRune(rune(i))
+		asciiTable[i] = p
+	}
+}
+
+func (t *Transformer) advance(s []byte) (n int, ok bool) {
+	var e bidi.Properties
+	var sz int
+	for n < len(s) {
+		if s[n] < utf8.RuneSelf {
+			e, sz = asciiTable[s[n]], 1
+		} else {
+			e, sz = bidi.Lookup(s[n:])
+			if sz <= 1 {
+				if sz == 1 {
+					// We always consider invalid UTF-8 to be invalid, even if
+					// the string has not yet been determined to be RTL.
+					// TODO: is this correct?
+					return n, false
+				}
+				return n, true // incomplete UTF-8 encoding
+			}
+		}
+		// TODO: using CompactClass would result in noticeable speedup.
+		// See unicode/bidi/prop.go:Properties.CompactClass.
+		c := uint16(1 << e.Class())
+		t.seen |= c
+		if t.seen&exclusiveRTL == exclusiveRTL {
+			t.state = ruleInvalid
+			return n, false
+		}
+		switch tr := transitions[t.state]; {
+		case tr[0].mask&c != 0:
+			t.state = tr[0].next
+		case tr[1].mask&c != 0:
+			t.state = tr[1].next
+		default:
+			t.state = ruleInvalid
+			if t.isRTL() {
+				return n, false
+			}
+		}
+		n += sz
+	}
+	return n, true
+}
+
+func (t *Transformer) advanceString(s string) (n int, ok bool) {
+	var e bidi.Properties
+	var sz int
+	for n < len(s) {
+		if s[n] < utf8.RuneSelf {
+			e, sz = asciiTable[s[n]], 1
+		} else {
+			e, sz = bidi.LookupString(s[n:])
+			if sz <= 1 {
+				if sz == 1 {
+					return n, false // invalid UTF-8
+				}
+				return n, true // incomplete UTF-8 encoding
+			}
+		}
+		// TODO: using CompactClass results in noticeable speedup.
+		// See unicode/bidi/prop.go:Properties.CompactClass.
+		c := uint16(1 << e.Class())
+		t.seen |= c
+		if t.seen&exclusiveRTL == exclusiveRTL {
+			t.state = ruleInvalid
+			return n, false
+		}
+		switch tr := transitions[t.state]; {
+		case tr[0].mask&c != 0:
+			t.state = tr[0].next
+		case tr[1].mask&c != 0:
+			t.state = tr[1].next
+		default:
+			t.state = ruleInvalid
+			if t.isRTL() {
+				return n, false
+			}
+		}
+		n += sz
+	}
+	return n, true
+}

vendor/golang.org/x/text/transform/transform.go

@@ -0,0 +1,705 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package transform provides reader and writer wrappers that transform the
+// bytes passing through as well as various transformations. Example
+// transformations provided by other packages include normalization and
+// conversion between character sets.
+package transform // import "golang.org/x/text/transform"
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"unicode/utf8"
+)
+
+var (
+	// ErrShortDst means that the destination buffer was too short to
+	// receive all of the transformed bytes.
+	ErrShortDst = errors.New("transform: short destination buffer")
+
+	// ErrShortSrc means that the source buffer has insufficient data to
+	// complete the transformation.
+	ErrShortSrc = errors.New("transform: short source buffer")
+
+	// ErrEndOfSpan means that the input and output (the transformed input)
+	// are not identical.
+	ErrEndOfSpan = errors.New("transform: input and output are not identical")
+
+	// errInconsistentByteCount means that Transform returned success (nil
+	// error) but also returned nSrc inconsistent with the src argument.
+	errInconsistentByteCount = errors.New("transform: inconsistent byte count returned")
+
+	// errShortInternal means that an internal buffer is not large enough
+	// to make progress and the Transform operation must be aborted.
+	errShortInternal = errors.New("transform: short internal buffer")
+)
+
+// Transformer transforms bytes.
+type Transformer interface {
+	// Transform writes to dst the transformed bytes read from src, and
+	// returns the number of dst bytes written and src bytes read. The
+	// atEOF argument tells whether src represents the last bytes of the
+	// input.
+	//
+	// Callers should always process the nDst bytes produced and account
+	// for the nSrc bytes consumed before considering the error err.
+	//
+	// A nil error means that all of the transformed bytes (whether freshly
+	// transformed from src or left over from previous Transform calls)
+	// were written to dst. A nil error can be returned regardless of
+	// whether atEOF is true. If err is nil then nSrc must equal len(src);
+	// the converse is not necessarily true.
+	//
+	// ErrShortDst means that dst was too short to receive all of the
+	// transformed bytes. ErrShortSrc means that src had insufficient data
+	// to complete the transformation. If both conditions apply, then
+	// either error may be returned. Other than the error conditions listed
+	// here, implementations are free to report other errors that arise.
+	Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
+
+	// Reset resets the state and allows a Transformer to be reused.
+	Reset()
+}
+
+// SpanningTransformer extends the Transformer interface with a Span method
+// that determines how much of the input already conforms to the Transformer.
+type SpanningTransformer interface {
+	Transformer
+
+	// Span returns a position in src such that transforming src[:n] results in
+	// identical output src[:n] for these bytes. It does not necessarily return
+	// the largest such n. The atEOF argument tells whether src represents the
+	// last bytes of the input.
+	//
+	// Callers should always account for the n bytes consumed before
+	// considering the error err.
+	//
+	// A nil error means that all input bytes are known to be identical to the
+	// output produced by the Transformer. A nil error can be be returned
+	// regardless of whether atEOF is true. If err is nil, then then n must
+	// equal len(src); the converse is not necessarily true.
+	//
+	// ErrEndOfSpan means that the Transformer output may differ from the
+	// input after n bytes. Note that n may be len(src), meaning that the output
+	// would contain additional bytes after otherwise identical output.
+	// ErrShortSrc means that src had insufficient data to determine whether the
+	// remaining bytes would change. Other than the error conditions listed
+	// here, implementations are free to report other errors that arise.
+	//
+	// Calling Span can modify the Transformer state as a side effect. In
+	// effect, it does the transformation just as calling Transform would, only
+	// without copying to a destination buffer and only up to a point it can
+	// determine the input and output bytes are the same. This is obviously more
+	// limited than calling Transform, but can be more efficient in terms of
+	// copying and allocating buffers. Calls to Span and Transform may be
+	// interleaved.
+	Span(src []byte, atEOF bool) (n int, err error)
+}
+
+// NopResetter can be embedded by implementations of Transformer to add a nop
+// Reset method.
+type NopResetter struct{}
+
+// Reset implements the Reset method of the Transformer interface.
+func (NopResetter) Reset() {}
+
+// Reader wraps another io.Reader by transforming the bytes read.
+type Reader struct {
+	r   io.Reader
+	t   Transformer
+	err error
+
+	// dst[dst0:dst1] contains bytes that have been transformed by t but
+	// not yet copied out via Read.
+	dst        []byte
+	dst0, dst1 int
+
+	// src[src0:src1] contains bytes that have been read from r but not
+	// yet transformed through t.
+	src        []byte
+	src0, src1 int
+
+	// transformComplete is whether the transformation is complete,
+	// regardless of whether or not it was successful.
+	transformComplete bool
+}
+
+const defaultBufSize = 4096
+
+// NewReader returns a new Reader that wraps r by transforming the bytes read
+// via t. It calls Reset on t.
+func NewReader(r io.Reader, t Transformer) *Reader {
+	t.Reset()
+	return &Reader{
+		r:   r,
+		t:   t,
+		dst: make([]byte, defaultBufSize),
+		src: make([]byte, defaultBufSize),
+	}
+}
+
+// Read implements the io.Reader interface.
+func (r *Reader) Read(p []byte) (int, error) {
+	n, err := 0, error(nil)
+	for {
+		// Copy out any transformed bytes and return the final error if we are done.
+		if r.dst0 != r.dst1 {
+			n = copy(p, r.dst[r.dst0:r.dst1])
+			r.dst0 += n
+			if r.dst0 == r.dst1 && r.transformComplete {
+				return n, r.err
+			}
+			return n, nil
+		} else if r.transformComplete {
+			return 0, r.err
+		}
+
+		// Try to transform some source bytes, or to flush the transformer if we
+		// are out of source bytes. We do this even if r.r.Read returned an error.
+		// As the io.Reader documentation says, "process the n > 0 bytes returned
+		// before considering the error".
+		if r.src0 != r.src1 || r.err != nil {
+			r.dst0 = 0
+			r.dst1, n, err = r.t.Transform(r.dst, r.src[r.src0:r.src1], r.err == io.EOF)
+			r.src0 += n
+
+			switch {
+			case err == nil:
+				if r.src0 != r.src1 {
+					r.err = errInconsistentByteCount
+				}
+				// The Transform call was successful; we are complete if we
+				// cannot read more bytes into src.
+				r.transformComplete = r.err != nil
+				continue
+			case err == ErrShortDst && (r.dst1 != 0 || n != 0):
+				// Make room in dst by copying out, and try again.
+				continue
+			case err == ErrShortSrc && r.src1-r.src0 != len(r.src) && r.err == nil:
+				// Read more bytes into src via the code below, and try again.
+			default:
+				r.transformComplete = true
+				// The reader error (r.err) takes precedence over the
+				// transformer error (err) unless r.err is nil or io.EOF.
+				if r.err == nil || r.err == io.EOF {
+					r.err = err
+				}
+				continue
+			}
+		}
+
+		// Move any untransformed source bytes to the start of the buffer
+		// and read more bytes.
+		if r.src0 != 0 {
+			r.src0, r.src1 = 0, copy(r.src, r.src[r.src0:r.src1])
+		}
+		n, r.err = r.r.Read(r.src[r.src1:])
+		r.src1 += n
+	}
+}
+
+// TODO: implement ReadByte (and ReadRune??).
+
+// Writer wraps another io.Writer by transforming the bytes read.
+// The user needs to call Close to flush unwritten bytes that may
+// be buffered.
+type Writer struct {
+	w   io.Writer
+	t   Transformer
+	dst []byte
+
+	// src[:n] contains bytes that have not yet passed through t.
+	src []byte
+	n   int
+}
+
+// NewWriter returns a new Writer that wraps w by transforming the bytes written
+// via t. It calls Reset on t.
+func NewWriter(w io.Writer, t Transformer) *Writer {
+	t.Reset()
+	return &Writer{
+		w:   w,
+		t:   t,
+		dst: make([]byte, defaultBufSize),
+		src: make([]byte, defaultBufSize),
+	}
+}
+
+// Write implements the io.Writer interface. If there are not enough
+// bytes available to complete a Transform, the bytes will be buffered
+// for the next write. Call Close to convert the remaining bytes.
+func (w *Writer) Write(data []byte) (n int, err error) {
+	src := data
+	if w.n > 0 {
+		// Append bytes from data to the last remainder.
+		// TODO: limit the amount copied on first try.
+		n = copy(w.src[w.n:], data)
+		w.n += n
+		src = w.src[:w.n]
+	}
+	for {
+		nDst, nSrc, err := w.t.Transform(w.dst, src, false)
+		if _, werr := w.w.Write(w.dst[:nDst]); werr != nil {
+			return n, werr
+		}
+		src = src[nSrc:]
+		if w.n == 0 {
+			n += nSrc
+		} else if len(src) <= n {
+			// Enough bytes from w.src have been consumed. We make src point
+			// to data instead to reduce the copying.
+			w.n = 0
+			n -= len(src)
+			src = data[n:]
+			if n < len(data) && (err == nil || err == ErrShortSrc) {
+				continue
+			}
+		}
+		switch err {
+		case ErrShortDst:
+			// This error is okay as long as we are making progress.
+			if nDst > 0 || nSrc > 0 {
+				continue
+			}
+		case ErrShortSrc:
+			if len(src) < len(w.src) {
+				m := copy(w.src, src)
+				// If w.n > 0, bytes from data were already copied to w.src and n
+				// was already set to the number of bytes consumed.
+				if w.n == 0 {
+					n += m
+				}
+				w.n = m
+				err = nil
+			} else if nDst > 0 || nSrc > 0 {
+				// Not enough buffer to store the remainder. Keep processing as
+				// long as there is progress. Without this case, transforms that
+				// require a lookahead larger than the buffer may result in an
+				// error. This is not something one may expect to be common in
+				// practice, but it may occur when buffers are set to small
+				// sizes during testing.
+				continue
+			}
+		case nil:
+			if w.n > 0 {
+				err = errInconsistentByteCount
+			}
+		}
+		return n, err
+	}
+}
+
+// Close implements the io.Closer interface.
+func (w *Writer) Close() error {
+	src := w.src[:w.n]
+	for {
+		nDst, nSrc, err := w.t.Transform(w.dst, src, true)
+		if _, werr := w.w.Write(w.dst[:nDst]); werr != nil {
+			return werr
+		}
+		if err != ErrShortDst {
+			return err
+		}
+		src = src[nSrc:]
+	}
+}
+
+type nop struct{ NopResetter }
+
+func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := copy(dst, src)
+	if n < len(src) {
+		err = ErrShortDst
+	}
+	return n, n, err
+}
+
+func (nop) Span(src []byte, atEOF bool) (n int, err error) {
+	return len(src), nil
+}
+
+type discard struct{ NopResetter }
+
+func (discard) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	return 0, len(src), nil
+}
+
+var (
+	// Discard is a Transformer for which all Transform calls succeed
+	// by consuming all bytes and writing nothing.
+	Discard Transformer = discard{}
+
+	// Nop is a SpanningTransformer that copies src to dst.
+	Nop SpanningTransformer = nop{}
+)
+
+// chain is a sequence of links. A chain with N Transformers has N+1 links and
+// N+1 buffers. Of those N+1 buffers, the first and last are the src and dst
+// buffers given to chain.Transform and the middle N-1 buffers are intermediate
+// buffers owned by the chain. The i'th link transforms bytes from the i'th
+// buffer chain.link[i].b at read offset chain.link[i].p to the i+1'th buffer
+// chain.link[i+1].b at write offset chain.link[i+1].n, for i in [0, N).
+type chain struct {
+	link []link
+	err  error
+	// errStart is the index at which the error occurred plus 1. Processing
+	// errStart at this level at the next call to Transform. As long as
+	// errStart > 0, chain will not consume any more source bytes.
+	errStart int
+}
+
+func (c *chain) fatalError(errIndex int, err error) {
+	if i := errIndex + 1; i > c.errStart {
+		c.errStart = i
+		c.err = err
+	}
+}
+
+type link struct {
+	t Transformer
+	// b[p:n] holds the bytes to be transformed by t.
+	b []byte
+	p int
+	n int
+}
+
+func (l *link) src() []byte {
+	return l.b[l.p:l.n]
+}
+
+func (l *link) dst() []byte {
+	return l.b[l.n:]
+}
+
+// Chain returns a Transformer that applies t in sequence.
+func Chain(t ...Transformer) Transformer {
+	if len(t) == 0 {
+		return nop{}
+	}
+	c := &chain{link: make([]link, len(t)+1)}
+	for i, tt := range t {
+		c.link[i].t = tt
+	}
+	// Allocate intermediate buffers.
+	b := make([][defaultBufSize]byte, len(t)-1)
+	for i := range b {
+		c.link[i+1].b = b[i][:]
+	}
+	return c
+}
+
+// Reset resets the state of Chain. It calls Reset on all the Transformers.
+func (c *chain) Reset() {
+	for i, l := range c.link {
+		if l.t != nil {
+			l.t.Reset()
+		}
+		c.link[i].p, c.link[i].n = 0, 0
+	}
+}
+
+// TODO: make chain use Span (is going to be fun to implement!)
+
+// Transform applies the transformers of c in sequence.
+func (c *chain) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// Set up src and dst in the chain.
+	srcL := &c.link[0]
+	dstL := &c.link[len(c.link)-1]
+	srcL.b, srcL.p, srcL.n = src, 0, len(src)
+	dstL.b, dstL.n = dst, 0
+	var lastFull, needProgress bool // for detecting progress
+
+	// i is the index of the next Transformer to apply, for i in [low, high].
+	// low is the lowest index for which c.link[low] may still produce bytes.
+	// high is the highest index for which c.link[high] has a Transformer.
+	// The error returned by Transform determines whether to increase or
+	// decrease i. We try to completely fill a buffer before converting it.
+	for low, i, high := c.errStart, c.errStart, len(c.link)-2; low <= i && i <= high; {
+		in, out := &c.link[i], &c.link[i+1]
+		nDst, nSrc, err0 := in.t.Transform(out.dst(), in.src(), atEOF && low == i)
+		out.n += nDst
+		in.p += nSrc
+		if i > 0 && in.p == in.n {
+			in.p, in.n = 0, 0
+		}
+		needProgress, lastFull = lastFull, false
+		switch err0 {
+		case ErrShortDst:
+			// Process the destination buffer next. Return if we are already
+			// at the high index.
+			if i == high {
+				return dstL.n, srcL.p, ErrShortDst
+			}
+			if out.n != 0 {
+				i++
+				// If the Transformer at the next index is not able to process any
+				// source bytes there is nothing that can be done to make progress
+				// and the bytes will remain unprocessed. lastFull is used to
+				// detect this and break out of the loop with a fatal error.
+				lastFull = true
+				continue
+			}
+			// The destination buffer was too small, but is completely empty.
+			// Return a fatal error as this transformation can never complete.
+			c.fatalError(i, errShortInternal)
+		case ErrShortSrc:
+			if i == 0 {
+				// Save ErrShortSrc in err. All other errors take precedence.
+				err = ErrShortSrc
+				break
+			}
+			// Source bytes were depleted before filling up the destination buffer.
+			// Verify we made some progress, move the remaining bytes to the errStart
+			// and try to get more source bytes.
+			if needProgress && nSrc == 0 || in.n-in.p == len(in.b) {
+				// There were not enough source bytes to proceed while the source
+				// buffer cannot hold any more bytes. Return a fatal error as this
+				// transformation can never complete.
+				c.fatalError(i, errShortInternal)
+				break
+			}
+			// in.b is an internal buffer and we can make progress.
+			in.p, in.n = 0, copy(in.b, in.src())
+			fallthrough
+		case nil:
+			// if i == low, we have depleted the bytes at index i or any lower levels.
+			// In that case we increase low and i. In all other cases we decrease i to
+			// fetch more bytes before proceeding to the next index.
+			if i > low {
+				i--
+				continue
+			}
+		default:
+			c.fatalError(i, err0)
+		}
+		// Exhausted level low or fatal error: increase low and continue
+		// to process the bytes accepted so far.
+		i++
+		low = i
+	}
+
+	// If c.errStart > 0, this means we found a fatal error.  We will clear
+	// all upstream buffers. At this point, no more progress can be made
+	// downstream, as Transform would have bailed while handling ErrShortDst.
+	if c.errStart > 0 {
+		for i := 1; i < c.errStart; i++ {
+			c.link[i].p, c.link[i].n = 0, 0
+		}
+		err, c.errStart, c.err = c.err, 0, nil
+	}
+	return dstL.n, srcL.p, err
+}
+
+// Deprecated: use runes.Remove instead.
+func RemoveFunc(f func(r rune) bool) Transformer {
+	return removeF(f)
+}
+
+type removeF func(r rune) bool
+
+func (removeF) Reset() {}
+
+// Transform implements the Transformer interface.
+func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for r, sz := rune(0), 0; len(src) > 0; src = src[sz:] {
+
+		if r = rune(src[0]); r < utf8.RuneSelf {
+			sz = 1
+		} else {
+			r, sz = utf8.DecodeRune(src)
+
+			if sz == 1 {
+				// Invalid rune.
+				if !atEOF && !utf8.FullRune(src) {
+					err = ErrShortSrc
+					break
+				}
+				// We replace illegal bytes with RuneError. Not doing so might
+				// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
+				// The resulting byte sequence may subsequently contain runes
+				// for which t(r) is true that were passed unnoticed.
+				if !t(r) {
+					if nDst+3 > len(dst) {
+						err = ErrShortDst
+						break
+					}
+					nDst += copy(dst[nDst:], "\uFFFD")
+				}
+				nSrc++
+				continue
+			}
+		}
+
+		if !t(r) {
+			if nDst+sz > len(dst) {
+				err = ErrShortDst
+				break
+			}
+			nDst += copy(dst[nDst:], src[:sz])
+		}
+		nSrc += sz
+	}
+	return
+}
+
+// grow returns a new []byte that is longer than b, and copies the first n bytes
+// of b to the start of the new slice.
+func grow(b []byte, n int) []byte {
+	m := len(b)
+	if m <= 32 {
+		m = 64
+	} else if m <= 256 {
+		m *= 2
+	} else {
+		m += m >> 1
+	}
+	buf := make([]byte, m)
+	copy(buf, b[:n])
+	return buf
+}
+
+const initialBufSize = 128
+
+// String returns a string with the result of converting s[:n] using t, where
+// n <= len(s). If err == nil, n will be len(s). It calls Reset on t.
+func String(t Transformer, s string) (result string, n int, err error) {
+	t.Reset()
+	if s == "" {
+		// Fast path for the common case for empty input. Results in about a
+		// 86% reduction of running time for BenchmarkStringLowerEmpty.
+		if _, _, err := t.Transform(nil, nil, true); err == nil {
+			return "", 0, nil
+		}
+	}
+
+	// Allocate only once. Note that both dst and src escape when passed to
+	// Transform.
+	buf := [2 * initialBufSize]byte{}
+	dst := buf[:initialBufSize:initialBufSize]
+	src := buf[initialBufSize : 2*initialBufSize]
+
+	// The input string s is transformed in multiple chunks (starting with a
+	// chunk size of initialBufSize). nDst and nSrc are per-chunk (or
+	// per-Transform-call) indexes, pDst and pSrc are overall indexes.
+	nDst, nSrc := 0, 0
+	pDst, pSrc := 0, 0
+
+	// pPrefix is the length of a common prefix: the first pPrefix bytes of the
+	// result will equal the first pPrefix bytes of s. It is not guaranteed to
+	// be the largest such value, but if pPrefix, len(result) and len(s) are
+	// all equal after the final transform (i.e. calling Transform with atEOF
+	// being true returned nil error) then we don't need to allocate a new
+	// result string.
+	pPrefix := 0
+	for {
+		// Invariant: pDst == pPrefix && pSrc == pPrefix.
+
+		n := copy(src, s[pSrc:])
+		nDst, nSrc, err = t.Transform(dst, src[:n], pSrc+n == len(s))
+		pDst += nDst
+		pSrc += nSrc
+
+		// TODO:  let transformers implement an optional Spanner interface, akin
+		// to norm's QuickSpan. This would even allow us to avoid any allocation.
+		if !bytes.Equal(dst[:nDst], src[:nSrc]) {
+			break
+		}
+		pPrefix = pSrc
+		if err == ErrShortDst {
+			// A buffer can only be short if a transformer modifies its input.
+			break
+		} else if err == ErrShortSrc {
+			if nSrc == 0 {
+				// No progress was made.
+				break
+			}
+			// Equal so far and !atEOF, so continue checking.
+		} else if err != nil || pPrefix == len(s) {
+			return string(s[:pPrefix]), pPrefix, err
+		}
+	}
+	// Post-condition: pDst == pPrefix + nDst && pSrc == pPrefix + nSrc.
+
+	// We have transformed the first pSrc bytes of the input s to become pDst
+	// transformed bytes. Those transformed bytes are discontiguous: the first
+	// pPrefix of them equal s[:pPrefix] and the last nDst of them equal
+	// dst[:nDst]. We copy them around, into a new dst buffer if necessary, so
+	// that they become one contiguous slice: dst[:pDst].
+	if pPrefix != 0 {
+		newDst := dst
+		if pDst > len(newDst) {
+			newDst = make([]byte, len(s)+nDst-nSrc)
+		}
+		copy(newDst[pPrefix:pDst], dst[:nDst])
+		copy(newDst[:pPrefix], s[:pPrefix])
+		dst = newDst
+	}
+
+	// Prevent duplicate Transform calls with atEOF being true at the end of
+	// the input. Also return if we have an unrecoverable error.
+	if (err == nil && pSrc == len(s)) ||
+		(err != nil && err != ErrShortDst && err != ErrShortSrc) {
+		return string(dst[:pDst]), pSrc, err
+	}
+
+	// Transform the remaining input, growing dst and src buffers as necessary.
+	for {
+		n := copy(src, s[pSrc:])
+		nDst, nSrc, err := t.Transform(dst[pDst:], src[:n], pSrc+n == len(s))
+		pDst += nDst
+		pSrc += nSrc
+
+		// If we got ErrShortDst or ErrShortSrc, do not grow as long as we can
+		// make progress. This may avoid excessive allocations.
+		if err == ErrShortDst {
+			if nDst == 0 {
+				dst = grow(dst, pDst)
+			}
+		} else if err == ErrShortSrc {
+			if nSrc == 0 {
+				src = grow(src, 0)
+			}
+		} else if err != nil || pSrc == len(s) {
+			return string(dst[:pDst]), pSrc, err
+		}
+	}
+}
+
+// Bytes returns a new byte slice with the result of converting b[:n] using t,
+// where n <= len(b). If err == nil, n will be len(b). It calls Reset on t.
+func Bytes(t Transformer, b []byte) (result []byte, n int, err error) {
+	return doAppend(t, 0, make([]byte, len(b)), b)
+}
+
+// Append appends the result of converting src[:n] using t to dst, where
+// n <= len(src), If err == nil, n will be len(src). It calls Reset on t.
+func Append(t Transformer, dst, src []byte) (result []byte, n int, err error) {
+	if len(dst) == cap(dst) {
+		n := len(src) + len(dst) // It is okay for this to be 0.
+		b := make([]byte, n)
+		dst = b[:copy(b, dst)]
+	}
+	return doAppend(t, len(dst), dst[:cap(dst)], src)
+}
+
+func doAppend(t Transformer, pDst int, dst, src []byte) (result []byte, n int, err error) {
+	t.Reset()
+	pSrc := 0
+	for {
+		nDst, nSrc, err := t.Transform(dst[pDst:], src[pSrc:], true)
+		pDst += nDst
+		pSrc += nSrc
+		if err != ErrShortDst {
+			return dst[:pDst], pSrc, err
+		}
+
+		// Grow the destination buffer, but do not grow as long as we can make
+		// progress. This may avoid excessive allocations.
+		if nDst == 0 {
+			dst = grow(dst, pDst)
+		}
+	}
+}

vendor/golang.org/x/text/unicode/bidi/bidi.go

@@ -0,0 +1,198 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go gen_trieval.go gen_ranges.go
+
+// Package bidi contains functionality for bidirectional text support.
+//
+// See http://www.unicode.org/reports/tr9.
+//
+// NOTE: UNDER CONSTRUCTION. This API may change in backwards incompatible ways
+// and without notice.
+package bidi // import "golang.org/x/text/unicode/bidi"
+
+// TODO:
+// The following functionality would not be hard to implement, but hinges on
+// the definition of a Segmenter interface. For now this is up to the user.
+// - Iterate over paragraphs
+// - Segmenter to iterate over runs directly from a given text.
+// Also:
+// - Transformer for reordering?
+// - Transformer (validator, really) for Bidi Rule.
+
+// This API tries to avoid dealing with embedding levels for now. Under the hood
+// these will be computed, but the question is to which extent the user should
+// know they exist. We should at some point allow the user to specify an
+// embedding hierarchy, though.
+
+// A Direction indicates the overall flow of text.
+type Direction int
+
+const (
+	// LeftToRight indicates the text contains no right-to-left characters and
+	// that either there are some left-to-right characters or the option
+	// DefaultDirection(LeftToRight) was passed.
+	LeftToRight Direction = iota
+
+	// RightToLeft indicates the text contains no left-to-right characters and
+	// that either there are some right-to-left characters or the option
+	// DefaultDirection(RightToLeft) was passed.
+	RightToLeft
+
+	// Mixed indicates text contains both left-to-right and right-to-left
+	// characters.
+	Mixed
+
+	// Neutral means that text contains no left-to-right and right-to-left
+	// characters and that no default direction has been set.
+	Neutral
+)
+
+type options struct{}
+
+// An Option is an option for Bidi processing.
+type Option func(*options)
+
+// ICU allows the user to define embedding levels. This may be used, for example,
+// to use hierarchical structure of markup languages to define embeddings.
+// The following option may be a way to expose this functionality in this API.
+// // LevelFunc sets a function that associates nesting levels with the given text.
+// // The levels function will be called with monotonically increasing values for p.
+// func LevelFunc(levels func(p int) int) Option {
+// 	panic("unimplemented")
+// }
+
+// DefaultDirection sets the default direction for a Paragraph. The direction is
+// overridden if the text contains directional characters.
+func DefaultDirection(d Direction) Option {
+	panic("unimplemented")
+}
+
+// A Paragraph holds a single Paragraph for Bidi processing.
+type Paragraph struct {
+	// buffers
+}
+
+// SetBytes configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetBytes(b []byte, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// SetString configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetString(s string, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// IsLeftToRight reports whether the principle direction of rendering for this
+// paragraphs is left-to-right. If this returns false, the principle direction
+// of rendering is right-to-left.
+func (p *Paragraph) IsLeftToRight() bool {
+	panic("unimplemented")
+}
+
+// Direction returns the direction of the text of this paragraph.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (p *Paragraph) Direction() Direction {
+	panic("unimplemented")
+}
+
+// RunAt reports the Run at the given position of the input text.
+//
+// This method can be used for computing line breaks on paragraphs.
+func (p *Paragraph) RunAt(pos int) Run {
+	panic("unimplemented")
+}
+
+// Order computes the visual ordering of all the runs in a Paragraph.
+func (p *Paragraph) Order() (Ordering, error) {
+	panic("unimplemented")
+}
+
+// Line computes the visual ordering of runs for a single line starting and
+// ending at the given positions in the original text.
+func (p *Paragraph) Line(start, end int) (Ordering, error) {
+	panic("unimplemented")
+}
+
+// An Ordering holds the computed visual order of runs of a Paragraph. Calling
+// SetBytes or SetString on the originating Paragraph invalidates an Ordering.
+// The methods of an Ordering should only be called by one goroutine at a time.
+type Ordering struct{}
+
+// Direction reports the directionality of the runs.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (o *Ordering) Direction() Direction {
+	panic("unimplemented")
+}
+
+// NumRuns returns the number of runs.
+func (o *Ordering) NumRuns() int {
+	panic("unimplemented")
+}
+
+// Run returns the ith run within the ordering.
+func (o *Ordering) Run(i int) Run {
+	panic("unimplemented")
+}
+
+// TODO: perhaps with options.
+// // Reorder creates a reader that reads the runes in visual order per character.
+// // Modifiers remain after the runes they modify.
+// func (l *Runs) Reorder() io.Reader {
+// 	panic("unimplemented")
+// }
+
+// A Run is a continuous sequence of characters of a single direction.
+type Run struct {
+}
+
+// String returns the text of the run in its original order.
+func (r *Run) String() string {
+	panic("unimplemented")
+}
+
+// Bytes returns the text of the run in its original order.
+func (r *Run) Bytes() []byte {
+	panic("unimplemented")
+}
+
+// TODO: methods for
+// - Display order
+// - headers and footers
+// - bracket replacement.
+
+// Direction reports the direction of the run.
+func (r *Run) Direction() Direction {
+	panic("unimplemented")
+}
+
+// Position of the Run within the text passed to SetBytes or SetString of the
+// originating Paragraph value.
+func (r *Run) Pos() (start, end int) {
+	panic("unimplemented")
+}
+
+// AppendReverse reverses the order of characters of in, appends them to out,
+// and returns the result. Modifiers will still follow the runes they modify.
+// Brackets are replaced with their counterparts.
+func AppendReverse(out, in []byte) []byte {
+	panic("unimplemented")
+}
+
+// ReverseString reverses the order of characters in s and returns a new string.
+// Modifiers will still follow the runes they modify. Brackets are replaced with
+// their counterparts.
+func ReverseString(s string) string {
+	panic("unimplemented")
+}

vendor/golang.org/x/text/unicode/bidi/bracket.go

@@ -0,0 +1,335 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bidi
+
+import (
+	"container/list"
+	"fmt"
+	"sort"
+)
+
+// This file contains a port of the reference implementation of the
+// Bidi Parentheses Algorithm:
+// http://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/BidiPBAReference.java
+//
+// The implementation in this file covers definitions BD14-BD16 and rule N0
+// of UAX#9.
+//
+// Some preprocessing is done for each rune before data is passed to this
+// algorithm:
+//  - opening and closing brackets are identified
+//  - a bracket pair type, like '(' and ')' is assigned a unique identifier that
+//    is identical for the opening and closing bracket. It is left to do these
+//    mappings.
+//  - The BPA algorithm requires that bracket characters that are canonical
+//    equivalents of each other be able to be substituted for each other.
+//    It is the responsibility of the caller to do this canonicalization.
+//
+// In implementing BD16, this implementation departs slightly from the "logical"
+// algorithm defined in UAX#9. In particular, the stack referenced there
+// supports operations that go beyond a "basic" stack. An equivalent
+// implementation based on a linked list is used here.
+
+// Bidi_Paired_Bracket_Type
+// BD14. An opening paired bracket is a character whose
+// Bidi_Paired_Bracket_Type property value is Open.
+//
+// BD15. A closing paired bracket is a character whose
+// Bidi_Paired_Bracket_Type property value is Close.
+type bracketType byte
+
+const (
+	bpNone bracketType = iota
+	bpOpen
+	bpClose
+)
+
+// bracketPair holds a pair of index values for opening and closing bracket
+// location of a bracket pair.
+type bracketPair struct {
+	opener int
+	closer int
+}
+
+func (b *bracketPair) String() string {
+	return fmt.Sprintf("(%v, %v)", b.opener, b.closer)
+}
+
+// bracketPairs is a slice of bracketPairs with a sort.Interface implementation.
+type bracketPairs []bracketPair
+
+func (b bracketPairs) Len() int           { return len(b) }
+func (b bracketPairs) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b bracketPairs) Less(i, j int) bool { return b[i].opener < b[j].opener }
+
+// resolvePairedBrackets runs the paired bracket part of the UBA algorithm.
+//
+// For each rune, it takes the indexes into the original string, the class the
+// bracket type (in pairTypes) and the bracket identifier (pairValues). It also
+// takes the direction type for the start-of-sentence and the embedding level.
+//
+// The identifiers for bracket types are the rune of the canonicalized opening
+// bracket for brackets (open or close) or 0 for runes that are not brackets.
+func resolvePairedBrackets(s *isolatingRunSequence) {
+	p := bracketPairer{
+		sos:              s.sos,
+		openers:          list.New(),
+		codesIsolatedRun: s.types,
+		indexes:          s.indexes,
+	}
+	dirEmbed := L
+	if s.level&1 != 0 {
+		dirEmbed = R
+	}
+	p.locateBrackets(s.p.pairTypes, s.p.pairValues)
+	p.resolveBrackets(dirEmbed, s.p.initialTypes)
+}
+
+type bracketPairer struct {
+	sos Class // direction corresponding to start of sequence
+
+	// The following is a restatement of BD 16 using non-algorithmic language.
+	//
+	// A bracket pair is a pair of characters consisting of an opening
+	// paired bracket and a closing paired bracket such that the
+	// Bidi_Paired_Bracket property value of the former equals the latter,
+	// subject to the following constraints.
+	// - both characters of a pair occur in the same isolating run sequence
+	// - the closing character of a pair follows the opening character
+	// - any bracket character can belong at most to one pair, the earliest possible one
+	// - any bracket character not part of a pair is treated like an ordinary character
+	// - pairs may nest properly, but their spans may not overlap otherwise
+
+	// Bracket characters with canonical decompositions are supposed to be
+	// treated as if they had been normalized, to allow normalized and non-
+	// normalized text to give the same result. In this implementation that step
+	// is pushed out to the caller. The caller has to ensure that the pairValue
+	// slices contain the rune of the opening bracket after normalization for
+	// any opening or closing bracket.
+
+	openers *list.List // list of positions for opening brackets
+
+	// bracket pair positions sorted by location of opening bracket
+	pairPositions bracketPairs
+
+	codesIsolatedRun []Class // directional bidi codes for an isolated run
+	indexes          []int   // array of index values into the original string
+
+}
+
+// matchOpener reports whether characters at given positions form a matching
+// bracket pair.
+func (p *bracketPairer) matchOpener(pairValues []rune, opener, closer int) bool {
+	return pairValues[p.indexes[opener]] == pairValues[p.indexes[closer]]
+}
+
+const maxPairingDepth = 63
+
+// locateBrackets locates matching bracket pairs according to BD16.
+//
+// This implementation uses a linked list instead of a stack, because, while
+// elements are added at the front (like a push) they are not generally removed
+// in atomic 'pop' operations, reducing the benefit of the stack archetype.
+func (p *bracketPairer) locateBrackets(pairTypes []bracketType, pairValues []rune) {
+	// traverse the run
+	// do that explicitly (not in a for-each) so we can record position
+	for i, index := range p.indexes {
+
+		// look at the bracket type for each character
+		if pairTypes[index] == bpNone || p.codesIsolatedRun[i] != ON {
+			// continue scanning
+			continue
+		}
+		switch pairTypes[index] {
+		case bpOpen:
+			// check if maximum pairing depth reached
+			if p.openers.Len() == maxPairingDepth {
+				p.openers.Init()
+				return
+			}
+			// remember opener location, most recent first
+			p.openers.PushFront(i)
+
+		case bpClose:
+			// see if there is a match
+			count := 0
+			for elem := p.openers.Front(); elem != nil; elem = elem.Next() {
+				count++
+				opener := elem.Value.(int)
+				if p.matchOpener(pairValues, opener, i) {
+					// if the opener matches, add nested pair to the ordered list
+					p.pairPositions = append(p.pairPositions, bracketPair{opener, i})
+					// remove up to and including matched opener
+					for ; count > 0; count-- {
+						p.openers.Remove(p.openers.Front())
+					}
+					break
+				}
+			}
+			sort.Sort(p.pairPositions)
+			// if we get here, the closing bracket matched no openers
+			// and gets ignored
+		}
+	}
+}
+
+// Bracket pairs within an isolating run sequence are processed as units so
+// that both the opening and the closing paired bracket in a pair resolve to
+// the same direction.
+//
+// N0. Process bracket pairs in an isolating run sequence sequentially in
+// the logical order of the text positions of the opening paired brackets
+// using the logic given below. Within this scope, bidirectional types EN
+// and AN are treated as R.
+//
+// Identify the bracket pairs in the current isolating run sequence
+// according to BD16. For each bracket-pair element in the list of pairs of
+// text positions:
+//
+// a Inspect the bidirectional types of the characters enclosed within the
+// bracket pair.
+//
+// b If any strong type (either L or R) matching the embedding direction is
+// found, set the type for both brackets in the pair to match the embedding
+// direction.
+//
+// o [ e ] o -> o e e e o
+//
+// o [ o e ] -> o e o e e
+//
+// o [ NI e ] -> o e NI e e
+//
+// c Otherwise, if a strong type (opposite the embedding direction) is
+// found, test for adjacent strong types as follows: 1 First, check
+// backwards before the opening paired bracket until the first strong type
+// (L, R, or sos) is found. If that first preceding strong type is opposite
+// the embedding direction, then set the type for both brackets in the pair
+// to that type. 2 Otherwise, set the type for both brackets in the pair to
+// the embedding direction.
+//
+// o [ o ] e -> o o o o e
+//
+// o [ o NI ] o -> o o o NI o o
+//
+// e [ o ] o -> e e o e o
+//
+// e [ o ] e -> e e o e e
+//
+// e ( o [ o ] NI ) e -> e e o o o o NI e e
+//
+// d Otherwise, do not set the type for the current bracket pair. Note that
+// if the enclosed text contains no strong types the paired brackets will
+// both resolve to the same level when resolved individually using rules N1
+// and N2.
+//
+// e ( NI ) o -> e ( NI ) o
+
+// getStrongTypeN0 maps character's directional code to strong type as required
+// by rule N0.
+//
+// TODO: have separate type for "strong" directionality.
+func (p *bracketPairer) getStrongTypeN0(index int) Class {
+	switch p.codesIsolatedRun[index] {
+	// in the scope of N0, number types are treated as R
+	case EN, AN, AL, R:
+		return R
+	case L:
+		return L
+	default:
+		return ON
+	}
+}
+
+// classifyPairContent reports the strong types contained inside a Bracket Pair,
+// assuming the given embedding direction.
+//
+// It returns ON if no strong type is found. If a single strong type is found,
+// it returns this this type. Otherwise it returns the embedding direction.
+//
+// TODO: use separate type for "strong" directionality.
+func (p *bracketPairer) classifyPairContent(loc bracketPair, dirEmbed Class) Class {
+	dirOpposite := ON
+	for i := loc.opener + 1; i < loc.closer; i++ {
+		dir := p.getStrongTypeN0(i)
+		if dir == ON {
+			continue
+		}
+		if dir == dirEmbed {
+			return dir // type matching embedding direction found
+		}
+		dirOpposite = dir
+	}
+	// return ON if no strong type found, or class opposite to dirEmbed
+	return dirOpposite
+}
+
+// classBeforePair determines which strong types are present before a Bracket
+// Pair. Return R or L if strong type found, otherwise ON.
+func (p *bracketPairer) classBeforePair(loc bracketPair) Class {
+	for i := loc.opener - 1; i >= 0; i-- {
+		if dir := p.getStrongTypeN0(i); dir != ON {
+			return dir
+		}
+	}
+	// no strong types found, return sos
+	return p.sos
+}
+
+// assignBracketType implements rule N0 for a single bracket pair.
+func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class, initialTypes []Class) {
+	// rule "N0, a", inspect contents of pair
+	dirPair := p.classifyPairContent(loc, dirEmbed)
+
+	// dirPair is now L, R, or N (no strong type found)
+
+	// the following logical tests are performed out of order compared to
+	// the statement of the rules but yield the same results
+	if dirPair == ON {
+		return // case "d" - nothing to do
+	}
+
+	if dirPair != dirEmbed {
+		// case "c": strong type found, opposite - check before (c.1)
+		dirPair = p.classBeforePair(loc)
+		if dirPair == dirEmbed || dirPair == ON {
+			// no strong opposite type found before - use embedding (c.2)
+			dirPair = dirEmbed
+		}
+	}
+	// else: case "b", strong type found matching embedding,
+	// no explicit action needed, as dirPair is already set to embedding
+	// direction
+
+	// set the bracket types to the type found
+	p.setBracketsToType(loc, dirPair, initialTypes)
+}
+
+func (p *bracketPairer) setBracketsToType(loc bracketPair, dirPair Class, initialTypes []Class) {
+	p.codesIsolatedRun[loc.opener] = dirPair
+	p.codesIsolatedRun[loc.closer] = dirPair
+
+	for i := loc.opener + 1; i < loc.closer; i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
+
+	for i := loc.closer + 1; i < len(p.indexes); i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
+}
+
+// resolveBrackets implements rule N0 for a list of pairs.
+func (p *bracketPairer) resolveBrackets(dirEmbed Class, initialTypes []Class) {
+	for _, loc := range p.pairPositions {
+		p.assignBracketType(loc, dirEmbed, initialTypes)
+	}
+}

vendor/golang.org/x/text/unicode/bidi/gen.go

@@ -0,0 +1,133 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"flag"
+	"log"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/triegen"
+	"golang.org/x/text/internal/ucd"
+)
+
+var outputFile = flag.String("out", "tables.go", "output file")
+
+func main() {
+	gen.Init()
+	gen.Repackage("gen_trieval.go", "trieval.go", "bidi")
+	gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi")
+
+	genTables()
+}
+
+// bidiClass names and codes taken from class "bc" in
+// http://www.unicode.org/Public/8.0.0/ucd/PropertyValueAliases.txt
+var bidiClass = map[string]Class{
+	"AL":  AL,  // ArabicLetter
+	"AN":  AN,  // ArabicNumber
+	"B":   B,   // ParagraphSeparator
+	"BN":  BN,  // BoundaryNeutral
+	"CS":  CS,  // CommonSeparator
+	"EN":  EN,  // EuropeanNumber
+	"ES":  ES,  // EuropeanSeparator
+	"ET":  ET,  // EuropeanTerminator
+	"L":   L,   // LeftToRight
+	"NSM": NSM, // NonspacingMark
+	"ON":  ON,  // OtherNeutral
+	"R":   R,   // RightToLeft
+	"S":   S,   // SegmentSeparator
+	"WS":  WS,  // WhiteSpace
+
+	"FSI": Control,
+	"PDF": Control,
+	"PDI": Control,
+	"LRE": Control,
+	"LRI": Control,
+	"LRO": Control,
+	"RLE": Control,
+	"RLI": Control,
+	"RLO": Control,
+}
+
+func genTables() {
+	if numClass > 0x0F {
+		log.Fatalf("Too many Class constants (%#x > 0x0F).", numClass)
+	}
+	w := gen.NewCodeWriter()
+	defer w.WriteGoFile(*outputFile, "bidi")
+
+	gen.WriteUnicodeVersion(w)
+
+	t := triegen.NewTrie("bidi")
+
+	// Build data about bracket mapping. These bits need to be or-ed with
+	// any other bits.
+	orMask := map[rune]uint64{}
+
+	xorMap := map[rune]int{}
+	xorMasks := []rune{0} // First value is no-op.
+
+	ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) {
+		r1 := p.Rune(0)
+		r2 := p.Rune(1)
+		xor := r1 ^ r2
+		if _, ok := xorMap[xor]; !ok {
+			xorMap[xor] = len(xorMasks)
+			xorMasks = append(xorMasks, xor)
+		}
+		entry := uint64(xorMap[xor]) << xorMaskShift
+		switch p.String(2) {
+		case "o":
+			entry |= openMask
+		case "c", "n":
+		default:
+			log.Fatalf("Unknown bracket class %q.", p.String(2))
+		}
+		orMask[r1] = entry
+	})
+
+	w.WriteComment(`
+	xorMasks contains masks to be xor-ed with brackets to get the reverse
+	version.`)
+	w.WriteVar("xorMasks", xorMasks)
+
+	done := map[rune]bool{}
+
+	insert := func(r rune, c Class) {
+		if !done[r] {
+			t.Insert(r, orMask[r]|uint64(c))
+			done[r] = true
+		}
+	}
+
+	// Insert the derived BiDi properties.
+	ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) {
+		r := p.Rune(0)
+		class, ok := bidiClass[p.String(1)]
+		if !ok {
+			log.Fatalf("%U: Unknown BiDi class %q", r, p.String(1))
+		}
+		insert(r, class)
+	})
+	visitDefaults(insert)
+
+	// TODO: use sparse blocks. This would reduce table size considerably
+	// from the looks of it.
+
+	sz, err := t.Gen(w)
+	if err != nil {
+		log.Fatal(err)
+	}
+	w.Size += sz
+}
+
+// dummy values to make methods in gen_common compile. The real versions
+// will be generated by this file to tables.go.
+var (
+	xorMasks []rune
+)

vendor/golang.org/x/text/unicode/bidi/gen_ranges.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"unicode"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/ucd"
+	"golang.org/x/text/unicode/rangetable"
+)
+
+// These tables are hand-extracted from:
+// http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt
+func visitDefaults(fn func(r rune, c Class)) {
+	// first write default values for ranges listed above.
+	visitRunes(fn, AL, []rune{
+		0x0600, 0x07BF, // Arabic
+		0x08A0, 0x08FF, // Arabic Extended-A
+		0xFB50, 0xFDCF, // Arabic Presentation Forms
+		0xFDF0, 0xFDFF,
+		0xFE70, 0xFEFF,
+		0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols
+	})
+	visitRunes(fn, R, []rune{
+		0x0590, 0x05FF, // Hebrew
+		0x07C0, 0x089F, // Nko et al.
+		0xFB1D, 0xFB4F,
+		0x00010800, 0x00010FFF, // Cypriot Syllabary et. al.
+		0x0001E800, 0x0001EDFF,
+		0x0001EF00, 0x0001EFFF,
+	})
+	visitRunes(fn, ET, []rune{ // European Terminator
+		0x20A0, 0x20Cf, // Currency symbols
+	})
+	rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) {
+		fn(r, BN) // Boundary Neutral
+	})
+	ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) {
+		if p.String(1) == "Default_Ignorable_Code_Point" {
+			fn(p.Rune(0), BN) // Boundary Neutral
+		}
+	})
+}
+
+func visitRunes(fn func(r rune, c Class), c Class, runes []rune) {
+	for i := 0; i < len(runes); i += 2 {
+		lo, hi := runes[i], runes[i+1]
+		for j := lo; j <= hi; j++ {
+			fn(j, c)
+		}
+	}
+}

vendor/golang.org/x/text/unicode/bidi/gen_trieval.go

@@ -0,0 +1,64 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// Class is the Unicode BiDi class. Each rune has a single class.
+type Class uint
+
+const (
+	L       Class = iota // LeftToRight
+	R                    // RightToLeft
+	EN                   // EuropeanNumber
+	ES                   // EuropeanSeparator
+	ET                   // EuropeanTerminator
+	AN                   // ArabicNumber
+	CS                   // CommonSeparator
+	B                    // ParagraphSeparator
+	S                    // SegmentSeparator
+	WS                   // WhiteSpace
+	ON                   // OtherNeutral
+	BN                   // BoundaryNeutral
+	NSM                  // NonspacingMark
+	AL                   // ArabicLetter
+	Control              // Control LRO - PDI
+
+	numClass
+
+	LRO // LeftToRightOverride
+	RLO // RightToLeftOverride
+	LRE // LeftToRightEmbedding
+	RLE // RightToLeftEmbedding
+	PDF // PopDirectionalFormat
+	LRI // LeftToRightIsolate
+	RLI // RightToLeftIsolate
+	FSI // FirstStrongIsolate
+	PDI // PopDirectionalIsolate
+
+	unknownClass = ^Class(0)
+)
+
+var controlToClass = map[rune]Class{
+	0x202D: LRO, // LeftToRightOverride,
+	0x202E: RLO, // RightToLeftOverride,
+	0x202A: LRE, // LeftToRightEmbedding,
+	0x202B: RLE, // RightToLeftEmbedding,
+	0x202C: PDF, // PopDirectionalFormat,
+	0x2066: LRI, // LeftToRightIsolate,
+	0x2067: RLI, // RightToLeftIsolate,
+	0x2068: FSI, // FirstStrongIsolate,
+	0x2069: PDI, // PopDirectionalIsolate,
+}
+
+// A trie entry has the following bits:
+// 7..5  XOR mask for brackets
+// 4     1: Bracket open, 0: Bracket close
+// 3..0  Class type
+
+const (
+	openMask     = 0x10
+	xorMaskShift = 5
+)

vendor/golang.org/x/text/unicode/bidi/prop.go

@@ -0,0 +1,206 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bidi
+
+import "unicode/utf8"
+
+// Properties provides access to BiDi properties of runes.
+type Properties struct {
+	entry uint8
+	last  uint8
+}
+
+var trie = newBidiTrie(0)
+
+// TODO: using this for bidirule reduces the running time by about 5%. Consider
+// if this is worth exposing or if we can find a way to speed up the Class
+// method.
+//
+// // CompactClass is like Class, but maps all of the BiDi control classes
+// // (LRO, RLO, LRE, RLE, PDF, LRI, RLI, FSI, PDI) to the class Control.
+// func (p Properties) CompactClass() Class {
+// 	return Class(p.entry & 0x0F)
+// }
+
+// Class returns the Bidi class for p.
+func (p Properties) Class() Class {
+	c := Class(p.entry & 0x0F)
+	if c == Control {
+		c = controlByteToClass[p.last&0xF]
+	}
+	return c
+}
+
+// IsBracket reports whether the rune is a bracket.
+func (p Properties) IsBracket() bool { return p.entry&0xF0 != 0 }
+
+// IsOpeningBracket reports whether the rune is an opening bracket.
+// IsBracket must return true.
+func (p Properties) IsOpeningBracket() bool { return p.entry&openMask != 0 }
+
+// TODO: find a better API and expose.
+func (p Properties) reverseBracket(r rune) rune {
+	return xorMasks[p.entry>>xorMaskShift] ^ r
+}
+
+var controlByteToClass = [16]Class{
+	0xD: LRO, // U+202D LeftToRightOverride,
+	0xE: RLO, // U+202E RightToLeftOverride,
+	0xA: LRE, // U+202A LeftToRightEmbedding,
+	0xB: RLE, // U+202B RightToLeftEmbedding,
+	0xC: PDF, // U+202C PopDirectionalFormat,
+	0x6: LRI, // U+2066 LeftToRightIsolate,
+	0x7: RLI, // U+2067 RightToLeftIsolate,
+	0x8: FSI, // U+2068 FirstStrongIsolate,
+	0x9: PDI, // U+2069 PopDirectionalIsolate,
+}
+
+// LookupRune returns properties for r.
+func LookupRune(r rune) (p Properties, size int) {
+	var buf [4]byte
+	n := utf8.EncodeRune(buf[:], r)
+	return Lookup(buf[:n])
+}
+
+// TODO: these lookup methods are based on the generated trie code. The returned
+// sizes have slightly different semantics from the generated code, in that it
+// always returns size==1 for an illegal UTF-8 byte (instead of the length
+// of the maximum invalid subsequence). Most Transformers, like unicode/norm,
+// leave invalid UTF-8 untouched, in which case it has performance benefits to
+// do so (without changing the semantics). Bidi requires the semantics used here
+// for the bidirule implementation to be compatible with the Go semantics.
+//  They ultimately should perhaps be adopted by all trie implementations, for
+// convenience sake.
+// This unrolled code also boosts performance of the secure/bidirule package by
+// about 30%.
+// So, to remove this code:
+//   - add option to trie generator to define return type.
+//   - always return 1 byte size for ill-formed UTF-8 runes.
+
+// Lookup returns properties for the first rune in s and the width in bytes of
+// its encoding. The size will be 0 if s does not hold enough bytes to complete
+// the encoding.
+func Lookup(s []byte) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}
+
+// LookupString returns properties for the first rune in s and the width in
+// bytes of its encoding. The size will be 0 if s does not hold enough bytes to
+// complete the encoding.
+func LookupString(s string) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}

vendor/golang.org/x/text/unicode/bidi/trieval.go

@@ -0,0 +1,60 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package bidi
+
+// Class is the Unicode BiDi class. Each rune has a single class.
+type Class uint
+
+const (
+	L       Class = iota // LeftToRight
+	R                    // RightToLeft
+	EN                   // EuropeanNumber
+	ES                   // EuropeanSeparator
+	ET                   // EuropeanTerminator
+	AN                   // ArabicNumber
+	CS                   // CommonSeparator
+	B                    // ParagraphSeparator
+	S                    // SegmentSeparator
+	WS                   // WhiteSpace
+	ON                   // OtherNeutral
+	BN                   // BoundaryNeutral
+	NSM                  // NonspacingMark
+	AL                   // ArabicLetter
+	Control              // Control LRO - PDI
+
+	numClass
+
+	LRO // LeftToRightOverride
+	RLO // RightToLeftOverride
+	LRE // LeftToRightEmbedding
+	RLE // RightToLeftEmbedding
+	PDF // PopDirectionalFormat
+	LRI // LeftToRightIsolate
+	RLI // RightToLeftIsolate
+	FSI // FirstStrongIsolate
+	PDI // PopDirectionalIsolate
+
+	unknownClass = ^Class(0)
+)
+
+var controlToClass = map[rune]Class{
+	0x202D: LRO, // LeftToRightOverride,
+	0x202E: RLO, // RightToLeftOverride,
+	0x202A: LRE, // LeftToRightEmbedding,
+	0x202B: RLE, // RightToLeftEmbedding,
+	0x202C: PDF, // PopDirectionalFormat,
+	0x2066: LRI, // LeftToRightIsolate,
+	0x2067: RLI, // RightToLeftIsolate,
+	0x2068: FSI, // FirstStrongIsolate,
+	0x2069: PDI, // PopDirectionalIsolate,
+}
+
+// A trie entry has the following bits:
+// 7..5  XOR mask for brackets
+// 4     1: Bracket open, 0: Bracket close
+// 3..0  Class type
+
+const (
+	openMask     = 0x10
+	xorMaskShift = 5
+)

vendor/golang.org/x/text/unicode/cldr/base.go

@@ -0,0 +1,100 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"encoding/xml"
+	"regexp"
+	"strconv"
+)
+
+// Elem is implemented by every XML element.
+type Elem interface {
+	setEnclosing(Elem)
+	setName(string)
+	enclosing() Elem
+
+	GetCommon() *Common
+}
+
+type hidden struct {
+	CharData string `xml:",chardata"`
+	Alias    *struct {
+		Common
+		Source string `xml:"source,attr"`
+		Path   string `xml:"path,attr"`
+	} `xml:"alias"`
+	Def *struct {
+		Common
+		Choice string `xml:"choice,attr,omitempty"`
+		Type   string `xml:"type,attr,omitempty"`
+	} `xml:"default"`
+}
+
+// Common holds several of the most common attributes and sub elements
+// of an XML element.
+type Common struct {
+	XMLName         xml.Name
+	name            string
+	enclElem        Elem
+	Type            string `xml:"type,attr,omitempty"`
+	Reference       string `xml:"reference,attr,omitempty"`
+	Alt             string `xml:"alt,attr,omitempty"`
+	ValidSubLocales string `xml:"validSubLocales,attr,omitempty"`
+	Draft           string `xml:"draft,attr,omitempty"`
+	hidden
+}
+
+// Default returns the default type to select from the enclosed list
+// or "" if no default value is specified.
+func (e *Common) Default() string {
+	if e.Def == nil {
+		return ""
+	}
+	if e.Def.Choice != "" {
+		return e.Def.Choice
+	} else if e.Def.Type != "" {
+		// Type is still used by the default element in collation.
+		return e.Def.Type
+	}
+	return ""
+}
+
+// GetCommon returns e. It is provided such that Common implements Elem.
+func (e *Common) GetCommon() *Common {
+	return e
+}
+
+// Data returns the character data accumulated for this element.
+func (e *Common) Data() string {
+	e.CharData = charRe.ReplaceAllStringFunc(e.CharData, replaceUnicode)
+	return e.CharData
+}
+
+func (e *Common) setName(s string) {
+	e.name = s
+}
+
+func (e *Common) enclosing() Elem {
+	return e.enclElem
+}
+
+func (e *Common) setEnclosing(en Elem) {
+	e.enclElem = en
+}
+
+// Escape characters that can be escaped without further escaping the string.
+var charRe = regexp.MustCompile(`&#x[0-9a-fA-F]*;|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\x[0-9a-fA-F]{2}|\\[0-7]{3}|\\[abtnvfr]`)
+
+// replaceUnicode converts hexadecimal Unicode codepoint notations to a one-rune string.
+// It assumes the input string is correctly formatted.
+func replaceUnicode(s string) string {
+	if s[1] == '#' {
+		r, _ := strconv.ParseInt(s[3:len(s)-1], 16, 32)
+		return string(r)
+	}
+	r, _, _, _ := strconv.UnquoteChar(s, 0)
+	return string(r)
+}

vendor/golang.org/x/text/unicode/cldr/cldr.go

@@ -0,0 +1,130 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run makexml.go -output xml.go
+
+// Package cldr provides a parser for LDML and related XML formats.
+// This package is intended to be used by the table generation tools
+// for the various internationalization-related packages.
+// As the XML types are generated from the CLDR DTD, and as the CLDR standard
+// is periodically amended, this package may change considerably over time.
+// This mostly means that data may appear and disappear between versions.
+// That is, old code should keep compiling for newer versions, but data
+// may have moved or changed.
+// CLDR version 22 is the first version supported by this package.
+// Older versions may not work.
+package cldr // import "golang.org/x/text/unicode/cldr"
+
+import (
+	"fmt"
+	"sort"
+)
+
+// CLDR provides access to parsed data of the Unicode Common Locale Data Repository.
+type CLDR struct {
+	parent   map[string][]string
+	locale   map[string]*LDML
+	resolved map[string]*LDML
+	bcp47    *LDMLBCP47
+	supp     *SupplementalData
+}
+
+func makeCLDR() *CLDR {
+	return &CLDR{
+		parent:   make(map[string][]string),
+		locale:   make(map[string]*LDML),
+		resolved: make(map[string]*LDML),
+		bcp47:    &LDMLBCP47{},
+		supp:     &SupplementalData{},
+	}
+}
+
+// BCP47 returns the parsed BCP47 LDML data. If no such data was parsed, nil is returned.
+func (cldr *CLDR) BCP47() *LDMLBCP47 {
+	return nil
+}
+
+// Draft indicates the draft level of an element.
+type Draft int
+
+const (
+	Approved Draft = iota
+	Contributed
+	Provisional
+	Unconfirmed
+)
+
+var drafts = []string{"unconfirmed", "provisional", "contributed", "approved", ""}
+
+// ParseDraft returns the Draft value corresponding to the given string. The
+// empty string corresponds to Approved.
+func ParseDraft(level string) (Draft, error) {
+	if level == "" {
+		return Approved, nil
+	}
+	for i, s := range drafts {
+		if level == s {
+			return Unconfirmed - Draft(i), nil
+		}
+	}
+	return Approved, fmt.Errorf("cldr: unknown draft level %q", level)
+}
+
+func (d Draft) String() string {
+	return drafts[len(drafts)-1-int(d)]
+}
+
+// SetDraftLevel sets which draft levels to include in the evaluated LDML.
+// Any draft element for which the draft level is higher than lev will be excluded.
+// If multiple draft levels are available for a single element, the one with the
+// lowest draft level will be selected, unless preferDraft is true, in which case
+// the highest draft will be chosen.
+// It is assumed that the underlying LDML is canonicalized.
+func (cldr *CLDR) SetDraftLevel(lev Draft, preferDraft bool) {
+	// TODO: implement
+	cldr.resolved = make(map[string]*LDML)
+}
+
+// RawLDML returns the LDML XML for id in unresolved form.
+// id must be one of the strings returned by Locales.
+func (cldr *CLDR) RawLDML(loc string) *LDML {
+	return cldr.locale[loc]
+}
+
+// LDML returns the fully resolved LDML XML for loc, which must be one of
+// the strings returned by Locales.
+func (cldr *CLDR) LDML(loc string) (*LDML, error) {
+	return cldr.resolve(loc)
+}
+
+// Supplemental returns the parsed supplemental data. If no such data was parsed,
+// nil is returned.
+func (cldr *CLDR) Supplemental() *SupplementalData {
+	return cldr.supp
+}
+
+// Locales returns the locales for which there exist files.
+// Valid sublocales for which there is no file are not included.
+// The root locale is always sorted first.
+func (cldr *CLDR) Locales() []string {
+	loc := []string{"root"}
+	hasRoot := false
+	for l, _ := range cldr.locale {
+		if l == "root" {
+			hasRoot = true
+			continue
+		}
+		loc = append(loc, l)
+	}
+	sort.Strings(loc[1:])
+	if !hasRoot {
+		return loc[1:]
+	}
+	return loc
+}
+
+// Get fills in the fields of x based on the XPath path.
+func Get(e Elem, path string) (res Elem, err error) {
+	return walkXPath(e, path)
+}

vendor/golang.org/x/text/unicode/cldr/collate.go

@@ -0,0 +1,359 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"bufio"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// RuleProcessor can be passed to Collator's Process method, which
+// parses the rules and calls the respective method for each rule found.
+type RuleProcessor interface {
+	Reset(anchor string, before int) error
+	Insert(level int, str, context, extend string) error
+	Index(id string)
+}
+
+const (
+	// cldrIndex is a Unicode-reserved sentinel value used to mark the start
+	// of a grouping within an index.
+	// We ignore any rule that starts with this rune.
+	// See http://unicode.org/reports/tr35/#Collation_Elements for details.
+	cldrIndex = "\uFDD0"
+
+	// specialAnchor is the format in which to represent logical reset positions,
+	// such as "first tertiary ignorable".
+	specialAnchor = "<%s/>"
+)
+
+// Process parses the rules for the tailorings of this collation
+// and calls the respective methods of p for each rule found.
+func (c Collation) Process(p RuleProcessor) (err error) {
+	if len(c.Cr) > 0 {
+		if len(c.Cr) > 1 {
+			return fmt.Errorf("multiple cr elements, want 0 or 1")
+		}
+		return processRules(p, c.Cr[0].Data())
+	}
+	if c.Rules.Any != nil {
+		return c.processXML(p)
+	}
+	return errors.New("no tailoring data")
+}
+
+// processRules parses rules in the Collation Rule Syntax defined in
+// http://www.unicode.org/reports/tr35/tr35-collation.html#Collation_Tailorings.
+func processRules(p RuleProcessor, s string) (err error) {
+	chk := func(s string, e error) string {
+		if err == nil {
+			err = e
+		}
+		return s
+	}
+	i := 0 // Save the line number for use after the loop.
+	scanner := bufio.NewScanner(strings.NewReader(s))
+	for ; scanner.Scan() && err == nil; i++ {
+		for s := skipSpace(scanner.Text()); s != "" && s[0] != '#'; s = skipSpace(s) {
+			level := 5
+			var ch byte
+			switch ch, s = s[0], s[1:]; ch {
+			case '&': // followed by <anchor> or '[' <key> ']'
+				if s = skipSpace(s); consume(&s, '[') {
+					s = chk(parseSpecialAnchor(p, s))
+				} else {
+					s = chk(parseAnchor(p, 0, s))
+				}
+			case '<': // sort relation '<'{1,4}, optionally followed by '*'.
+				for level = 1; consume(&s, '<'); level++ {
+				}
+				if level > 4 {
+					err = fmt.Errorf("level %d > 4", level)
+				}
+				fallthrough
+			case '=': // identity relation, optionally followed by *.
+				if consume(&s, '*') {
+					s = chk(parseSequence(p, level, s))
+				} else {
+					s = chk(parseOrder(p, level, s))
+				}
+			default:
+				chk("", fmt.Errorf("illegal operator %q", ch))
+				break
+			}
+		}
+	}
+	if chk("", scanner.Err()); err != nil {
+		return fmt.Errorf("%d: %v", i, err)
+	}
+	return nil
+}
+
+// parseSpecialAnchor parses the anchor syntax which is either of the form
+//    ['before' <level>] <anchor>
+// or
+//    [<label>]
+// The starting should already be consumed.
+func parseSpecialAnchor(p RuleProcessor, s string) (tail string, err error) {
+	i := strings.IndexByte(s, ']')
+	if i == -1 {
+		return "", errors.New("unmatched bracket")
+	}
+	a := strings.TrimSpace(s[:i])
+	s = s[i+1:]
+	if strings.HasPrefix(a, "before ") {
+		l, err := strconv.ParseUint(skipSpace(a[len("before "):]), 10, 3)
+		if err != nil {
+			return s, err
+		}
+		return parseAnchor(p, int(l), s)
+	}
+	return s, p.Reset(fmt.Sprintf(specialAnchor, a), 0)
+}
+
+func parseAnchor(p RuleProcessor, level int, s string) (tail string, err error) {
+	anchor, s, err := scanString(s)
+	if err != nil {
+		return s, err
+	}
+	return s, p.Reset(anchor, level)
+}
+
+func parseOrder(p RuleProcessor, level int, s string) (tail string, err error) {
+	var value, context, extend string
+	if value, s, err = scanString(s); err != nil {
+		return s, err
+	}
+	if strings.HasPrefix(value, cldrIndex) {
+		p.Index(value[len(cldrIndex):])
+		return
+	}
+	if consume(&s, '|') {
+		if context, s, err = scanString(s); err != nil {
+			return s, errors.New("missing string after context")
+		}
+	}
+	if consume(&s, '/') {
+		if extend, s, err = scanString(s); err != nil {
+			return s, errors.New("missing string after extension")
+		}
+	}
+	return s, p.Insert(level, value, context, extend)
+}
+
+// scanString scans a single input string.
+func scanString(s string) (str, tail string, err error) {
+	if s = skipSpace(s); s == "" {
+		return s, s, errors.New("missing string")
+	}
+	buf := [16]byte{} // small but enough to hold most cases.
+	value := buf[:0]
+	for s != "" {
+		if consume(&s, '\'') {
+			i := strings.IndexByte(s, '\'')
+			if i == -1 {
+				return "", "", errors.New(`unmatched single quote`)
+			}
+			if i == 0 {
+				value = append(value, '\'')
+			} else {
+				value = append(value, s[:i]...)
+			}
+			s = s[i+1:]
+			continue
+		}
+		r, sz := utf8.DecodeRuneInString(s)
+		if unicode.IsSpace(r) || strings.ContainsRune("&<=#", r) {
+			break
+		}
+		value = append(value, s[:sz]...)
+		s = s[sz:]
+	}
+	return string(value), skipSpace(s), nil
+}
+
+func parseSequence(p RuleProcessor, level int, s string) (tail string, err error) {
+	if s = skipSpace(s); s == "" {
+		return s, errors.New("empty sequence")
+	}
+	last := rune(0)
+	for s != "" {
+		r, sz := utf8.DecodeRuneInString(s)
+		s = s[sz:]
+
+		if r == '-' {
+			// We have a range. The first element was already written.
+			if last == 0 {
+				return s, errors.New("range without starter value")
+			}
+			r, sz = utf8.DecodeRuneInString(s)
+			s = s[sz:]
+			if r == utf8.RuneError || r < last {
+				return s, fmt.Errorf("invalid range %q-%q", last, r)
+			}
+			for i := last + 1; i <= r; i++ {
+				if err := p.Insert(level, string(i), "", ""); err != nil {
+					return s, err
+				}
+			}
+			last = 0
+			continue
+		}
+
+		if unicode.IsSpace(r) || unicode.IsPunct(r) {
+			break
+		}
+
+		// normal case
+		if err := p.Insert(level, string(r), "", ""); err != nil {
+			return s, err
+		}
+		last = r
+	}
+	return s, nil
+}
+
+func skipSpace(s string) string {
+	return strings.TrimLeftFunc(s, unicode.IsSpace)
+}
+
+// consumes returns whether the next byte is ch. If so, it gobbles it by
+// updating s.
+func consume(s *string, ch byte) (ok bool) {
+	if *s == "" || (*s)[0] != ch {
+		return false
+	}
+	*s = (*s)[1:]
+	return true
+}
+
+// The following code parses Collation rules of CLDR version 24 and before.
+
+var lmap = map[byte]int{
+	'p': 1,
+	's': 2,
+	't': 3,
+	'i': 5,
+}
+
+type rulesElem struct {
+	Rules struct {
+		Common
+		Any []*struct {
+			XMLName xml.Name
+			rule
+		} `xml:",any"`
+	} `xml:"rules"`
+}
+
+type rule struct {
+	Value  string `xml:",chardata"`
+	Before string `xml:"before,attr"`
+	Any    []*struct {
+		XMLName xml.Name
+		rule
+	} `xml:",any"`
+}
+
+var emptyValueError = errors.New("cldr: empty rule value")
+
+func (r *rule) value() (string, error) {
+	// Convert hexadecimal Unicode codepoint notation to a string.
+	s := charRe.ReplaceAllStringFunc(r.Value, replaceUnicode)
+	r.Value = s
+	if s == "" {
+		if len(r.Any) != 1 {
+			return "", emptyValueError
+		}
+		r.Value = fmt.Sprintf(specialAnchor, r.Any[0].XMLName.Local)
+		r.Any = nil
+	} else if len(r.Any) != 0 {
+		return "", fmt.Errorf("cldr: XML elements found in collation rule: %v", r.Any)
+	}
+	return r.Value, nil
+}
+
+func (r rule) process(p RuleProcessor, name, context, extend string) error {
+	v, err := r.value()
+	if err != nil {
+		return err
+	}
+	switch name {
+	case "p", "s", "t", "i":
+		if strings.HasPrefix(v, cldrIndex) {
+			p.Index(v[len(cldrIndex):])
+			return nil
+		}
+		if err := p.Insert(lmap[name[0]], v, context, extend); err != nil {
+			return err
+		}
+	case "pc", "sc", "tc", "ic":
+		level := lmap[name[0]]
+		for _, s := range v {
+			if err := p.Insert(level, string(s), context, extend); err != nil {
+				return err
+			}
+		}
+	default:
+		return fmt.Errorf("cldr: unsupported tag: %q", name)
+	}
+	return nil
+}
+
+// processXML parses the format of CLDR versions 24 and older.
+func (c Collation) processXML(p RuleProcessor) (err error) {
+	// Collation is generated and defined in xml.go.
+	var v string
+	for _, r := range c.Rules.Any {
+		switch r.XMLName.Local {
+		case "reset":
+			level := 0
+			switch r.Before {
+			case "primary", "1":
+				level = 1
+			case "secondary", "2":
+				level = 2
+			case "tertiary", "3":
+				level = 3
+			case "":
+			default:
+				return fmt.Errorf("cldr: unknown level %q", r.Before)
+			}
+			v, err = r.value()
+			if err == nil {
+				err = p.Reset(v, level)
+			}
+		case "x":
+			var context, extend string
+			for _, r1 := range r.Any {
+				v, err = r1.value()
+				switch r1.XMLName.Local {
+				case "context":
+					context = v
+				case "extend":
+					extend = v
+				}
+			}
+			for _, r1 := range r.Any {
+				if t := r1.XMLName.Local; t == "context" || t == "extend" {
+					continue
+				}
+				r1.rule.process(p, r1.XMLName.Local, context, extend)
+			}
+		default:
+			err = r.rule.process(p, r.XMLName.Local, "", "")
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/golang.org/x/text/unicode/cldr/decode.go

@@ -0,0 +1,171 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+)
+
+// A Decoder loads an archive of CLDR data.
+type Decoder struct {
+	dirFilter     []string
+	sectionFilter []string
+	loader        Loader
+	cldr          *CLDR
+	curLocale     string
+}
+
+// SetSectionFilter takes a list top-level LDML element names to which
+// evaluation of LDML should be limited.  It automatically calls SetDirFilter.
+func (d *Decoder) SetSectionFilter(filter ...string) {
+	d.sectionFilter = filter
+	// TODO: automatically set dir filter
+}
+
+// SetDirFilter limits the loading of LDML XML files of the specied directories.
+// Note that sections may be split across directories differently for different CLDR versions.
+// For more robust code, use SetSectionFilter.
+func (d *Decoder) SetDirFilter(dir ...string) {
+	d.dirFilter = dir
+}
+
+// A Loader provides access to the files of a CLDR archive.
+type Loader interface {
+	Len() int
+	Path(i int) string
+	Reader(i int) (io.ReadCloser, error)
+}
+
+var fileRe = regexp.MustCompile(".*/(.*)/(.*)\\.xml")
+
+// Decode loads and decodes the files represented by l.
+func (d *Decoder) Decode(l Loader) (cldr *CLDR, err error) {
+	d.cldr = makeCLDR()
+	for i := 0; i < l.Len(); i++ {
+		fname := l.Path(i)
+		if m := fileRe.FindStringSubmatch(fname); m != nil {
+			if len(d.dirFilter) > 0 && !in(d.dirFilter, m[1]) {
+				continue
+			}
+			var r io.Reader
+			if r, err = l.Reader(i); err == nil {
+				err = d.decode(m[1], m[2], r)
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	d.cldr.finalize(d.sectionFilter)
+	return d.cldr, nil
+}
+
+func (d *Decoder) decode(dir, id string, r io.Reader) error {
+	var v interface{}
+	var l *LDML
+	cldr := d.cldr
+	switch {
+	case dir == "supplemental":
+		v = cldr.supp
+	case dir == "transforms":
+		return nil
+	case dir == "bcp47":
+		v = cldr.bcp47
+	case dir == "validity":
+		return nil
+	default:
+		ok := false
+		if v, ok = cldr.locale[id]; !ok {
+			l = &LDML{}
+			v, cldr.locale[id] = l, l
+		}
+	}
+	x := xml.NewDecoder(r)
+	if err := x.Decode(v); err != nil {
+		log.Printf("%s/%s: %v", dir, id, err)
+		return err
+	}
+	if l != nil {
+		if l.Identity == nil {
+			return fmt.Errorf("%s/%s: missing identity element", dir, id)
+		}
+		// TODO: verify when CLDR bug http://unicode.org/cldr/trac/ticket/8970
+		// is resolved.
+		// path := strings.Split(id, "_")
+		// if lang := l.Identity.Language.Type; lang != path[0] {
+		// 	return fmt.Errorf("%s/%s: language was %s; want %s", dir, id, lang, path[0])
+		// }
+	}
+	return nil
+}
+
+type pathLoader []string
+
+func makePathLoader(path string) (pl pathLoader, err error) {
+	err = filepath.Walk(path, func(path string, _ os.FileInfo, err error) error {
+		pl = append(pl, path)
+		return err
+	})
+	return pl, err
+}
+
+func (pl pathLoader) Len() int {
+	return len(pl)
+}
+
+func (pl pathLoader) Path(i int) string {
+	return pl[i]
+}
+
+func (pl pathLoader) Reader(i int) (io.ReadCloser, error) {
+	return os.Open(pl[i])
+}
+
+// DecodePath loads CLDR data from the given path.
+func (d *Decoder) DecodePath(path string) (cldr *CLDR, err error) {
+	loader, err := makePathLoader(path)
+	if err != nil {
+		return nil, err
+	}
+	return d.Decode(loader)
+}
+
+type zipLoader struct {
+	r *zip.Reader
+}
+
+func (zl zipLoader) Len() int {
+	return len(zl.r.File)
+}
+
+func (zl zipLoader) Path(i int) string {
+	return zl.r.File[i].Name
+}
+
+func (zl zipLoader) Reader(i int) (io.ReadCloser, error) {
+	return zl.r.File[i].Open()
+}
+
+// DecodeZip loads CLDR data from the zip archive for which r is the source.
+func (d *Decoder) DecodeZip(r io.Reader) (cldr *CLDR, err error) {
+	buffer, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	if err != nil {
+		return nil, err
+	}
+	return d.Decode(zipLoader{archive})
+}

vendor/golang.org/x/text/unicode/cldr/makexml.go

@@ -0,0 +1,400 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// This tool generates types for the various XML formats of CLDR.
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"golang.org/x/text/internal/gen"
+)
+
+var outputFile = flag.String("output", "xml.go", "output file name")
+
+func main() {
+	flag.Parse()
+
+	r := gen.OpenCLDRCoreZip()
+	buffer, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Fatal("Could not read zip file")
+	}
+	r.Close()
+	z, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	if err != nil {
+		log.Fatalf("Could not read zip archive: %v", err)
+	}
+
+	var buf bytes.Buffer
+
+	version := gen.CLDRVersion()
+
+	for _, dtd := range files {
+		for _, f := range z.File {
+			if strings.HasSuffix(f.Name, dtd.file+".dtd") {
+				r, err := f.Open()
+				failOnError(err)
+
+				b := makeBuilder(&buf, dtd)
+				b.parseDTD(r)
+				b.resolve(b.index[dtd.top[0]])
+				b.write()
+				if b.version != "" && version != b.version {
+					println(f.Name)
+					log.Fatalf("main: inconsistent versions: found %s; want %s", b.version, version)
+				}
+				break
+			}
+		}
+	}
+	fmt.Fprintln(&buf, "// Version is the version of CLDR from which the XML definitions are generated.")
+	fmt.Fprintf(&buf, "const Version = %q\n", version)
+
+	gen.WriteGoFile(*outputFile, "cldr", buf.Bytes())
+}
+
+func failOnError(err error) {
+	if err != nil {
+		log.New(os.Stderr, "", log.Lshortfile).Output(2, err.Error())
+		os.Exit(1)
+	}
+}
+
+// configuration data per DTD type
+type dtd struct {
+	file string   // base file name
+	root string   // Go name of the root XML element
+	top  []string // create a different type for this section
+
+	skipElem    []string // hard-coded or deprecated elements
+	skipAttr    []string // attributes to exclude
+	predefined  []string // hard-coded elements exist of the form <name>Elem
+	forceRepeat []string // elements to make slices despite DTD
+}
+
+var files = []dtd{
+	{
+		file: "ldmlBCP47",
+		root: "LDMLBCP47",
+		top:  []string{"ldmlBCP47"},
+		skipElem: []string{
+			"cldrVersion", // deprecated, not used
+		},
+	},
+	{
+		file: "ldmlSupplemental",
+		root: "SupplementalData",
+		top:  []string{"supplementalData"},
+		skipElem: []string{
+			"cldrVersion", // deprecated, not used
+		},
+		forceRepeat: []string{
+			"plurals", // data defined in plurals.xml and ordinals.xml
+		},
+	},
+	{
+		file: "ldml",
+		root: "LDML",
+		top: []string{
+			"ldml", "collation", "calendar", "timeZoneNames", "localeDisplayNames", "numbers",
+		},
+		skipElem: []string{
+			"cp",       // not used anywhere
+			"special",  // not used anywhere
+			"fallback", // deprecated, not used
+			"alias",    // in Common
+			"default",  // in Common
+		},
+		skipAttr: []string{
+			"hiraganaQuarternary", // typo in DTD, correct version included as well
+		},
+		predefined: []string{"rules"},
+	},
+}
+
+var comments = map[string]string{
+	"ldmlBCP47": `
+// LDMLBCP47 holds information on allowable values for various variables in LDML.
+`,
+	"supplementalData": `
+// SupplementalData holds information relevant for internationalization
+// and proper use of CLDR, but that is not contained in the locale hierarchy.
+`,
+	"ldml": `
+// LDML is the top-level type for locale-specific data.
+`,
+	"collation": `
+// Collation contains rules that specify a certain sort-order,
+// as a tailoring of the root order. 
+// The parsed rules are obtained by passing a RuleProcessor to Collation's
+// Process method.
+`,
+	"calendar": `
+// Calendar specifies the fields used for formatting and parsing dates and times.
+// The month and quarter names are identified numerically, starting at 1.
+// The day (of the week) names are identified with short strings, since there is
+// no universally-accepted numeric designation.
+`,
+	"dates": `
+// Dates contains information regarding the format and parsing of dates and times.
+`,
+	"localeDisplayNames": `
+// LocaleDisplayNames specifies localized display names for for scripts, languages,
+// countries, currencies, and variants.
+`,
+	"numbers": `
+// Numbers supplies information for formatting and parsing numbers and currencies.
+`,
+}
+
+type element struct {
+	name      string // XML element name
+	category  string // elements contained by this element
+	signature string // category + attrKey*
+
+	attr []*attribute // attributes supported by this element.
+	sub  []struct {   // parsed and evaluated sub elements of this element.
+		e      *element
+		repeat bool // true if the element needs to be a slice
+	}
+
+	resolved bool // prevent multiple resolutions of this element.
+}
+
+type attribute struct {
+	name string
+	key  string
+	list []string
+
+	tag string // Go tag
+}
+
+var (
+	reHead  = regexp.MustCompile(` *(\w+) +([\w\-]+)`)
+	reAttr  = regexp.MustCompile(` *(\w+) *(?:(\w+)|\(([\w\- \|]+)\)) *(?:#([A-Z]*) *(?:\"([\.\d+])\")?)? *("[\w\-:]*")?`)
+	reElem  = regexp.MustCompile(`^ *(EMPTY|ANY|\(.*\)[\*\+\?]?) *$`)
+	reToken = regexp.MustCompile(`\w\-`)
+)
+
+// builder is used to read in the DTD files from CLDR and generate Go code
+// to be used with the encoding/xml package.
+type builder struct {
+	w       io.Writer
+	index   map[string]*element
+	elem    []*element
+	info    dtd
+	version string
+}
+
+func makeBuilder(w io.Writer, d dtd) builder {
+	return builder{
+		w:     w,
+		index: make(map[string]*element),
+		elem:  []*element{},
+		info:  d,
+	}
+}
+
+// parseDTD parses a DTD file.
+func (b *builder) parseDTD(r io.Reader) {
+	for d := xml.NewDecoder(r); ; {
+		t, err := d.Token()
+		if t == nil {
+			break
+		}
+		failOnError(err)
+		dir, ok := t.(xml.Directive)
+		if !ok {
+			continue
+		}
+		m := reHead.FindSubmatch(dir)
+		dir = dir[len(m[0]):]
+		ename := string(m[2])
+		el, elementFound := b.index[ename]
+		switch string(m[1]) {
+		case "ELEMENT":
+			if elementFound {
+				log.Fatal("parseDTD: duplicate entry for element %q", ename)
+			}
+			m := reElem.FindSubmatch(dir)
+			if m == nil {
+				log.Fatalf("parseDTD: invalid element %q", string(dir))
+			}
+			if len(m[0]) != len(dir) {
+				log.Fatal("parseDTD: invalid element %q", string(dir), len(dir), len(m[0]), string(m[0]))
+			}
+			s := string(m[1])
+			el = &element{
+				name:     ename,
+				category: s,
+			}
+			b.index[ename] = el
+		case "ATTLIST":
+			if !elementFound {
+				log.Fatalf("parseDTD: unknown element %q", ename)
+			}
+			s := string(dir)
+			m := reAttr.FindStringSubmatch(s)
+			if m == nil {
+				log.Fatal(fmt.Errorf("parseDTD: invalid attribute %q", string(dir)))
+			}
+			if m[4] == "FIXED" {
+				b.version = m[5]
+			} else {
+				switch m[1] {
+				case "draft", "references", "alt", "validSubLocales", "standard" /* in Common */ :
+				case "type", "choice":
+				default:
+					el.attr = append(el.attr, &attribute{
+						name: m[1],
+						key:  s,
+						list: reToken.FindAllString(m[3], -1),
+					})
+					el.signature = fmt.Sprintf("%s=%s+%s", el.signature, m[1], m[2])
+				}
+			}
+		}
+	}
+}
+
+var reCat = regexp.MustCompile(`[ ,\|]*(?:(\(|\)|\#?[\w_-]+)([\*\+\?]?))?`)
+
+// resolve takes a parsed element and converts it into structured data
+// that can be used to generate the XML code.
+func (b *builder) resolve(e *element) {
+	if e.resolved {
+		return
+	}
+	b.elem = append(b.elem, e)
+	e.resolved = true
+	s := e.category
+	found := make(map[string]bool)
+	sequenceStart := []int{}
+	for len(s) > 0 {
+		m := reCat.FindStringSubmatch(s)
+		if m == nil {
+			log.Fatalf("%s: invalid category string %q", e.name, s)
+		}
+		repeat := m[2] == "*" || m[2] == "+" || in(b.info.forceRepeat, m[1])
+		switch m[1] {
+		case "":
+		case "(":
+			sequenceStart = append(sequenceStart, len(e.sub))
+		case ")":
+			if len(sequenceStart) == 0 {
+				log.Fatalf("%s: unmatched closing parenthesis", e.name)
+			}
+			for i := sequenceStart[len(sequenceStart)-1]; i < len(e.sub); i++ {
+				e.sub[i].repeat = e.sub[i].repeat || repeat
+			}
+			sequenceStart = sequenceStart[:len(sequenceStart)-1]
+		default:
+			if in(b.info.skipElem, m[1]) {
+			} else if sub, ok := b.index[m[1]]; ok {
+				if !found[sub.name] {
+					e.sub = append(e.sub, struct {
+						e      *element
+						repeat bool
+					}{sub, repeat})
+					found[sub.name] = true
+					b.resolve(sub)
+				}
+			} else if m[1] == "#PCDATA" || m[1] == "ANY" {
+			} else if m[1] != "EMPTY" {
+				log.Fatalf("resolve:%s: element %q not found", e.name, m[1])
+			}
+		}
+		s = s[len(m[0]):]
+	}
+}
+
+// return true if s is contained in set.
+func in(set []string, s string) bool {
+	for _, v := range set {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+var repl = strings.NewReplacer("-", " ", "_", " ")
+
+// title puts the first character or each character following '_' in title case and
+// removes all occurrences of '_'.
+func title(s string) string {
+	return strings.Replace(strings.Title(repl.Replace(s)), " ", "", -1)
+}
+
+// writeElem generates Go code for a single element, recursively.
+func (b *builder) writeElem(tab int, e *element) {
+	p := func(f string, x ...interface{}) {
+		f = strings.Replace(f, "\n", "\n"+strings.Repeat("\t", tab), -1)
+		fmt.Fprintf(b.w, f, x...)
+	}
+	if len(e.sub) == 0 && len(e.attr) == 0 {
+		p("Common")
+		return
+	}
+	p("struct {")
+	tab++
+	p("\nCommon")
+	for _, attr := range e.attr {
+		if !in(b.info.skipAttr, attr.name) {
+			p("\n%s string `xml:\"%s,attr\"`", title(attr.name), attr.name)
+		}
+	}
+	for _, sub := range e.sub {
+		if in(b.info.predefined, sub.e.name) {
+			p("\n%sElem", sub.e.name)
+			continue
+		}
+		if in(b.info.skipElem, sub.e.name) {
+			continue
+		}
+		p("\n%s ", title(sub.e.name))
+		if sub.repeat {
+			p("[]")
+		}
+		p("*")
+		if in(b.info.top, sub.e.name) {
+			p(title(sub.e.name))
+		} else {
+			b.writeElem(tab, sub.e)
+		}
+		p(" `xml:\"%s\"`", sub.e.name)
+	}
+	tab--
+	p("\n}")
+}
+
+// write generates the Go XML code.
+func (b *builder) write() {
+	for i, name := range b.info.top {
+		e := b.index[name]
+		if e != nil {
+			fmt.Fprintf(b.w, comments[name])
+			name := title(e.name)
+			if i == 0 {
+				name = b.info.root
+			}
+			fmt.Fprintf(b.w, "type %s ", name)
+			b.writeElem(0, e)
+			fmt.Fprint(b.w, "\n")
+		}
+	}
+}

vendor/golang.org/x/text/unicode/cldr/resolve.go

@@ -0,0 +1,602 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+// This file implements the various inheritance constructs defined by LDML.
+// See http://www.unicode.org/reports/tr35/#Inheritance_and_Validity
+// for more details.
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// fieldIter iterates over fields in a struct. It includes
+// fields of embedded structs.
+type fieldIter struct {
+	v        reflect.Value
+	index, n []int
+}
+
+func iter(v reflect.Value) fieldIter {
+	if v.Kind() != reflect.Struct {
+		log.Panicf("value %v must be a struct", v)
+	}
+	i := fieldIter{
+		v:     v,
+		index: []int{0},
+		n:     []int{v.NumField()},
+	}
+	i.descent()
+	return i
+}
+
+func (i *fieldIter) descent() {
+	for f := i.field(); f.Anonymous && f.Type.NumField() > 0; f = i.field() {
+		i.index = append(i.index, 0)
+		i.n = append(i.n, f.Type.NumField())
+	}
+}
+
+func (i *fieldIter) done() bool {
+	return len(i.index) == 1 && i.index[0] >= i.n[0]
+}
+
+func skip(f reflect.StructField) bool {
+	return !f.Anonymous && (f.Name[0] < 'A' || f.Name[0] > 'Z')
+}
+
+func (i *fieldIter) next() {
+	for {
+		k := len(i.index) - 1
+		i.index[k]++
+		if i.index[k] < i.n[k] {
+			if !skip(i.field()) {
+				break
+			}
+		} else {
+			if k == 0 {
+				return
+			}
+			i.index = i.index[:k]
+			i.n = i.n[:k]
+		}
+	}
+	i.descent()
+}
+
+func (i *fieldIter) value() reflect.Value {
+	return i.v.FieldByIndex(i.index)
+}
+
+func (i *fieldIter) field() reflect.StructField {
+	return i.v.Type().FieldByIndex(i.index)
+}
+
+type visitor func(v reflect.Value) error
+
+var stopDescent = fmt.Errorf("do not recurse")
+
+func (f visitor) visit(x interface{}) error {
+	return f.visitRec(reflect.ValueOf(x))
+}
+
+// visit recursively calls f on all nodes in v.
+func (f visitor) visitRec(v reflect.Value) error {
+	if v.Kind() == reflect.Ptr {
+		if v.IsNil() {
+			return nil
+		}
+		return f.visitRec(v.Elem())
+	}
+	if err := f(v); err != nil {
+		if err == stopDescent {
+			return nil
+		}
+		return err
+	}
+	switch v.Kind() {
+	case reflect.Struct:
+		for i := iter(v); !i.done(); i.next() {
+			if err := f.visitRec(i.value()); err != nil {
+				return err
+			}
+		}
+	case reflect.Slice:
+		for i := 0; i < v.Len(); i++ {
+			if err := f.visitRec(v.Index(i)); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// getPath is used for error reporting purposes only.
+func getPath(e Elem) string {
+	if e == nil {
+		return "<nil>"
+	}
+	if e.enclosing() == nil {
+		return e.GetCommon().name
+	}
+	if e.GetCommon().Type == "" {
+		return fmt.Sprintf("%s.%s", getPath(e.enclosing()), e.GetCommon().name)
+	}
+	return fmt.Sprintf("%s.%s[type=%s]", getPath(e.enclosing()), e.GetCommon().name, e.GetCommon().Type)
+}
+
+// xmlName returns the xml name of the element or attribute
+func xmlName(f reflect.StructField) (name string, attr bool) {
+	tags := strings.Split(f.Tag.Get("xml"), ",")
+	for _, s := range tags {
+		attr = attr || s == "attr"
+	}
+	return tags[0], attr
+}
+
+func findField(v reflect.Value, key string) (reflect.Value, error) {
+	v = reflect.Indirect(v)
+	for i := iter(v); !i.done(); i.next() {
+		if n, _ := xmlName(i.field()); n == key {
+			return i.value(), nil
+		}
+	}
+	return reflect.Value{}, fmt.Errorf("cldr: no field %q in element %#v", key, v.Interface())
+}
+
+var xpathPart = regexp.MustCompile(`(\pL+)(?:\[@(\pL+)='([\w-]+)'\])?`)
+
+func walkXPath(e Elem, path string) (res Elem, err error) {
+	for _, c := range strings.Split(path, "/") {
+		if c == ".." {
+			if e = e.enclosing(); e == nil {
+				panic("path ..")
+				return nil, fmt.Errorf(`cldr: ".." moves past root in path %q`, path)
+			}
+			continue
+		} else if c == "" {
+			continue
+		}
+		m := xpathPart.FindStringSubmatch(c)
+		if len(m) == 0 || len(m[0]) != len(c) {
+			return nil, fmt.Errorf("cldr: syntax error in path component %q", c)
+		}
+		v, err := findField(reflect.ValueOf(e), m[1])
+		if err != nil {
+			return nil, err
+		}
+		switch v.Kind() {
+		case reflect.Slice:
+			i := 0
+			if m[2] != "" || v.Len() > 1 {
+				if m[2] == "" {
+					m[2] = "type"
+					if m[3] = e.GetCommon().Default(); m[3] == "" {
+						return nil, fmt.Errorf("cldr: type selector or default value needed for element %s", m[1])
+					}
+				}
+				for ; i < v.Len(); i++ {
+					vi := v.Index(i)
+					key, err := findField(vi.Elem(), m[2])
+					if err != nil {
+						return nil, err
+					}
+					key = reflect.Indirect(key)
+					if key.Kind() == reflect.String && key.String() == m[3] {
+						break
+					}
+				}
+			}
+			if i == v.Len() || v.Index(i).IsNil() {
+				return nil, fmt.Errorf("no %s found with %s==%s", m[1], m[2], m[3])
+			}
+			e = v.Index(i).Interface().(Elem)
+		case reflect.Ptr:
+			if v.IsNil() {
+				return nil, fmt.Errorf("cldr: element %q not found within element %q", m[1], e.GetCommon().name)
+			}
+			var ok bool
+			if e, ok = v.Interface().(Elem); !ok {
+				return nil, fmt.Errorf("cldr: %q is not an XML element", m[1])
+			} else if m[2] != "" || m[3] != "" {
+				return nil, fmt.Errorf("cldr: no type selector allowed for element %s", m[1])
+			}
+		default:
+			return nil, fmt.Errorf("cldr: %q is not an XML element", m[1])
+		}
+	}
+	return e, nil
+}
+
+const absPrefix = "//ldml/"
+
+func (cldr *CLDR) resolveAlias(e Elem, src, path string) (res Elem, err error) {
+	if src != "locale" {
+		if !strings.HasPrefix(path, absPrefix) {
+			return nil, fmt.Errorf("cldr: expected absolute path, found %q", path)
+		}
+		path = path[len(absPrefix):]
+		if e, err = cldr.resolve(src); err != nil {
+			return nil, err
+		}
+	}
+	return walkXPath(e, path)
+}
+
+func (cldr *CLDR) resolveAndMergeAlias(e Elem) error {
+	alias := e.GetCommon().Alias
+	if alias == nil {
+		return nil
+	}
+	a, err := cldr.resolveAlias(e, alias.Source, alias.Path)
+	if err != nil {
+		return fmt.Errorf("%v: error evaluating path %q: %v", getPath(e), alias.Path, err)
+	}
+	// Ensure alias node was already evaluated. TODO: avoid double evaluation.
+	err = cldr.resolveAndMergeAlias(a)
+	v := reflect.ValueOf(e).Elem()
+	for i := iter(reflect.ValueOf(a).Elem()); !i.done(); i.next() {
+		if vv := i.value(); vv.Kind() != reflect.Ptr || !vv.IsNil() {
+			if _, attr := xmlName(i.field()); !attr {
+				v.FieldByIndex(i.index).Set(vv)
+			}
+		}
+	}
+	return err
+}
+
+func (cldr *CLDR) aliasResolver() visitor {
+	return func(v reflect.Value) (err error) {
+		if e, ok := v.Addr().Interface().(Elem); ok {
+			err = cldr.resolveAndMergeAlias(e)
+			if err == nil && blocking[e.GetCommon().name] {
+				return stopDescent
+			}
+		}
+		return err
+	}
+}
+
+// elements within blocking elements do not inherit.
+// Taken from CLDR's supplementalMetaData.xml.
+var blocking = map[string]bool{
+	"identity":         true,
+	"supplementalData": true,
+	"cldrTest":         true,
+	"collation":        true,
+	"transform":        true,
+}
+
+// Distinguishing attributes affect inheritance; two elements with different
+// distinguishing attributes are treated as different for purposes of inheritance,
+// except when such attributes occur in the indicated elements.
+// Taken from CLDR's supplementalMetaData.xml.
+var distinguishing = map[string][]string{
+	"key":        nil,
+	"request_id": nil,
+	"id":         nil,
+	"registry":   nil,
+	"alt":        nil,
+	"iso4217":    nil,
+	"iso3166":    nil,
+	"mzone":      nil,
+	"from":       nil,
+	"to":         nil,
+	"type": []string{
+		"abbreviationFallback",
+		"default",
+		"mapping",
+		"measurementSystem",
+		"preferenceOrdering",
+	},
+	"numberSystem": nil,
+}
+
+func in(set []string, s string) bool {
+	for _, v := range set {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// attrKey computes a key based on the distinguishable attributes of
+// an element and it's values.
+func attrKey(v reflect.Value, exclude ...string) string {
+	parts := []string{}
+	ename := v.Interface().(Elem).GetCommon().name
+	v = v.Elem()
+	for i := iter(v); !i.done(); i.next() {
+		if name, attr := xmlName(i.field()); attr {
+			if except, ok := distinguishing[name]; ok && !in(exclude, name) && !in(except, ename) {
+				v := i.value()
+				if v.Kind() == reflect.Ptr {
+					v = v.Elem()
+				}
+				if v.IsValid() {
+					parts = append(parts, fmt.Sprintf("%s=%s", name, v.String()))
+				}
+			}
+		}
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, ";")
+}
+
+// Key returns a key for e derived from all distinguishing attributes
+// except those specified by exclude.
+func Key(e Elem, exclude ...string) string {
+	return attrKey(reflect.ValueOf(e), exclude...)
+}
+
+// linkEnclosing sets the enclosing element as well as the name
+// for all sub-elements of child, recursively.
+func linkEnclosing(parent, child Elem) {
+	child.setEnclosing(parent)
+	v := reflect.ValueOf(child).Elem()
+	for i := iter(v); !i.done(); i.next() {
+		vf := i.value()
+		if vf.Kind() == reflect.Slice {
+			for j := 0; j < vf.Len(); j++ {
+				linkEnclosing(child, vf.Index(j).Interface().(Elem))
+			}
+		} else if vf.Kind() == reflect.Ptr && !vf.IsNil() && vf.Elem().Kind() == reflect.Struct {
+			linkEnclosing(child, vf.Interface().(Elem))
+		}
+	}
+}
+
+func setNames(e Elem, name string) {
+	e.setName(name)
+	v := reflect.ValueOf(e).Elem()
+	for i := iter(v); !i.done(); i.next() {
+		vf := i.value()
+		name, _ = xmlName(i.field())
+		if vf.Kind() == reflect.Slice {
+			for j := 0; j < vf.Len(); j++ {
+				setNames(vf.Index(j).Interface().(Elem), name)
+			}
+		} else if vf.Kind() == reflect.Ptr && !vf.IsNil() && vf.Elem().Kind() == reflect.Struct {
+			setNames(vf.Interface().(Elem), name)
+		}
+	}
+}
+
+// deepCopy copies elements of v recursively.  All elements of v that may
+// be modified by inheritance are explicitly copied.
+func deepCopy(v reflect.Value) reflect.Value {
+	switch v.Kind() {
+	case reflect.Ptr:
+		if v.IsNil() || v.Elem().Kind() != reflect.Struct {
+			return v
+		}
+		nv := reflect.New(v.Elem().Type())
+		nv.Elem().Set(v.Elem())
+		deepCopyRec(nv.Elem(), v.Elem())
+		return nv
+	case reflect.Slice:
+		nv := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
+		for i := 0; i < v.Len(); i++ {
+			deepCopyRec(nv.Index(i), v.Index(i))
+		}
+		return nv
+	}
+	panic("deepCopy: must be called with pointer or slice")
+}
+
+// deepCopyRec is only called by deepCopy.
+func deepCopyRec(nv, v reflect.Value) {
+	if v.Kind() == reflect.Struct {
+		t := v.Type()
+		for i := 0; i < v.NumField(); i++ {
+			if name, attr := xmlName(t.Field(i)); name != "" && !attr {
+				deepCopyRec(nv.Field(i), v.Field(i))
+			}
+		}
+	} else {
+		nv.Set(deepCopy(v))
+	}
+}
+
+// newNode is used to insert a missing node during inheritance.
+func (cldr *CLDR) newNode(v, enc reflect.Value) reflect.Value {
+	n := reflect.New(v.Type())
+	for i := iter(v); !i.done(); i.next() {
+		if name, attr := xmlName(i.field()); name == "" || attr {
+			n.Elem().FieldByIndex(i.index).Set(i.value())
+		}
+	}
+	n.Interface().(Elem).GetCommon().setEnclosing(enc.Addr().Interface().(Elem))
+	return n
+}
+
+// v, parent must be pointers to struct
+func (cldr *CLDR) inheritFields(v, parent reflect.Value) (res reflect.Value, err error) {
+	t := v.Type()
+	nv := reflect.New(t)
+	nv.Elem().Set(v)
+	for i := iter(v); !i.done(); i.next() {
+		vf := i.value()
+		f := i.field()
+		name, attr := xmlName(f)
+		if name == "" || attr {
+			continue
+		}
+		pf := parent.FieldByIndex(i.index)
+		if blocking[name] {
+			if vf.IsNil() {
+				vf = pf
+			}
+			nv.Elem().FieldByIndex(i.index).Set(deepCopy(vf))
+			continue
+		}
+		switch f.Type.Kind() {
+		case reflect.Ptr:
+			if f.Type.Elem().Kind() == reflect.Struct {
+				if !vf.IsNil() {
+					if vf, err = cldr.inheritStructPtr(vf, pf); err != nil {
+						return reflect.Value{}, err
+					}
+					vf.Interface().(Elem).setEnclosing(nv.Interface().(Elem))
+					nv.Elem().FieldByIndex(i.index).Set(vf)
+				} else if !pf.IsNil() {
+					n := cldr.newNode(pf.Elem(), v)
+					if vf, err = cldr.inheritStructPtr(n, pf); err != nil {
+						return reflect.Value{}, err
+					}
+					vf.Interface().(Elem).setEnclosing(nv.Interface().(Elem))
+					nv.Elem().FieldByIndex(i.index).Set(vf)
+				}
+			}
+		case reflect.Slice:
+			vf, err := cldr.inheritSlice(nv.Elem(), vf, pf)
+			if err != nil {
+				return reflect.Zero(t), err
+			}
+			nv.Elem().FieldByIndex(i.index).Set(vf)
+		}
+	}
+	return nv, nil
+}
+
+func root(e Elem) *LDML {
+	for ; e.enclosing() != nil; e = e.enclosing() {
+	}
+	return e.(*LDML)
+}
+
+// inheritStructPtr first merges possible aliases in with v and then inherits
+// any underspecified elements from parent.
+func (cldr *CLDR) inheritStructPtr(v, parent reflect.Value) (r reflect.Value, err error) {
+	if !v.IsNil() {
+		e := v.Interface().(Elem).GetCommon()
+		alias := e.Alias
+		if alias == nil && !parent.IsNil() {
+			alias = parent.Interface().(Elem).GetCommon().Alias
+		}
+		if alias != nil {
+			a, err := cldr.resolveAlias(v.Interface().(Elem), alias.Source, alias.Path)
+			if a != nil {
+				if v, err = cldr.inheritFields(v.Elem(), reflect.ValueOf(a).Elem()); err != nil {
+					return reflect.Value{}, err
+				}
+			}
+		}
+		if !parent.IsNil() {
+			return cldr.inheritFields(v.Elem(), parent.Elem())
+		}
+	} else if parent.IsNil() {
+		panic("should not reach here")
+	}
+	return v, nil
+}
+
+// Must be slice of struct pointers.
+func (cldr *CLDR) inheritSlice(enc, v, parent reflect.Value) (res reflect.Value, err error) {
+	t := v.Type()
+	index := make(map[string]reflect.Value)
+	if !v.IsNil() {
+		for i := 0; i < v.Len(); i++ {
+			vi := v.Index(i)
+			key := attrKey(vi)
+			index[key] = vi
+		}
+	}
+	if !parent.IsNil() {
+		for i := 0; i < parent.Len(); i++ {
+			vi := parent.Index(i)
+			key := attrKey(vi)
+			if w, ok := index[key]; ok {
+				index[key], err = cldr.inheritStructPtr(w, vi)
+			} else {
+				n := cldr.newNode(vi.Elem(), enc)
+				index[key], err = cldr.inheritStructPtr(n, vi)
+			}
+			index[key].Interface().(Elem).setEnclosing(enc.Addr().Interface().(Elem))
+			if err != nil {
+				return v, err
+			}
+		}
+	}
+	keys := make([]string, 0, len(index))
+	for k, _ := range index {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	sl := reflect.MakeSlice(t, len(index), len(index))
+	for i, k := range keys {
+		sl.Index(i).Set(index[k])
+	}
+	return sl, nil
+}
+
+func parentLocale(loc string) string {
+	parts := strings.Split(loc, "_")
+	if len(parts) == 1 {
+		return "root"
+	}
+	parts = parts[:len(parts)-1]
+	key := strings.Join(parts, "_")
+	return key
+}
+
+func (cldr *CLDR) resolve(loc string) (res *LDML, err error) {
+	if r := cldr.resolved[loc]; r != nil {
+		return r, nil
+	}
+	x := cldr.RawLDML(loc)
+	if x == nil {
+		return nil, fmt.Errorf("cldr: unknown locale %q", loc)
+	}
+	var v reflect.Value
+	if loc == "root" {
+		x = deepCopy(reflect.ValueOf(x)).Interface().(*LDML)
+		linkEnclosing(nil, x)
+		err = cldr.aliasResolver().visit(x)
+	} else {
+		key := parentLocale(loc)
+		var parent *LDML
+		for ; cldr.locale[key] == nil; key = parentLocale(key) {
+		}
+		if parent, err = cldr.resolve(key); err != nil {
+			return nil, err
+		}
+		v, err = cldr.inheritFields(reflect.ValueOf(x).Elem(), reflect.ValueOf(parent).Elem())
+		x = v.Interface().(*LDML)
+		linkEnclosing(nil, x)
+	}
+	if err != nil {
+		return nil, err
+	}
+	cldr.resolved[loc] = x
+	return x, err
+}
+
+// finalize finalizes the initialization of the raw LDML structs.  It also
+// removed unwanted fields, as specified by filter, so that they will not
+// be unnecessarily evaluated.
+func (cldr *CLDR) finalize(filter []string) {
+	for _, x := range cldr.locale {
+		if filter != nil {
+			v := reflect.ValueOf(x).Elem()
+			t := v.Type()
+			for i := 0; i < v.NumField(); i++ {
+				f := t.Field(i)
+				name, _ := xmlName(f)
+				if name != "" && name != "identity" && !in(filter, name) {
+					v.Field(i).Set(reflect.Zero(f.Type))
+				}
+			}
+		}
+		linkEnclosing(nil, x) // for resolving aliases and paths
+		setNames(x, "ldml")
+	}
+}

vendor/golang.org/x/text/unicode/cldr/slice.go

@@ -0,0 +1,144 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+)
+
+// Slice provides utilities for modifying slices of elements.
+// It can be wrapped around any slice of which the element type implements
+// interface Elem.
+type Slice struct {
+	ptr reflect.Value
+	typ reflect.Type
+}
+
+// Value returns the reflect.Value of the underlying slice.
+func (s *Slice) Value() reflect.Value {
+	return s.ptr.Elem()
+}
+
+// MakeSlice wraps a pointer to a slice of Elems.
+// It replaces the array pointed to by the slice so that subsequent modifications
+// do not alter the data in a CLDR type.
+// It panics if an incorrect type is passed.
+func MakeSlice(slicePtr interface{}) Slice {
+	ptr := reflect.ValueOf(slicePtr)
+	if ptr.Kind() != reflect.Ptr {
+		panic(fmt.Sprintf("MakeSlice: argument must be pointer to slice, found %v", ptr.Type()))
+	}
+	sl := ptr.Elem()
+	if sl.Kind() != reflect.Slice {
+		panic(fmt.Sprintf("MakeSlice: argument must point to a slice, found %v", sl.Type()))
+	}
+	intf := reflect.TypeOf((*Elem)(nil)).Elem()
+	if !sl.Type().Elem().Implements(intf) {
+		panic(fmt.Sprintf("MakeSlice: element type of slice (%v) does not implement Elem", sl.Type().Elem()))
+	}
+	nsl := reflect.MakeSlice(sl.Type(), sl.Len(), sl.Len())
+	reflect.Copy(nsl, sl)
+	sl.Set(nsl)
+	return Slice{
+		ptr: ptr,
+		typ: sl.Type().Elem().Elem(),
+	}
+}
+
+func (s Slice) indexForAttr(a string) []int {
+	for i := iter(reflect.Zero(s.typ)); !i.done(); i.next() {
+		if n, _ := xmlName(i.field()); n == a {
+			return i.index
+		}
+	}
+	panic(fmt.Sprintf("MakeSlice: no attribute %q for type %v", a, s.typ))
+}
+
+// Filter filters s to only include elements for which fn returns true.
+func (s Slice) Filter(fn func(e Elem) bool) {
+	k := 0
+	sl := s.Value()
+	for i := 0; i < sl.Len(); i++ {
+		vi := sl.Index(i)
+		if fn(vi.Interface().(Elem)) {
+			sl.Index(k).Set(vi)
+			k++
+		}
+	}
+	sl.Set(sl.Slice(0, k))
+}
+
+// Group finds elements in s for which fn returns the same value and groups
+// them in a new Slice.
+func (s Slice) Group(fn func(e Elem) string) []Slice {
+	m := make(map[string][]reflect.Value)
+	sl := s.Value()
+	for i := 0; i < sl.Len(); i++ {
+		vi := sl.Index(i)
+		key := fn(vi.Interface().(Elem))
+		m[key] = append(m[key], vi)
+	}
+	keys := []string{}
+	for k, _ := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	res := []Slice{}
+	for _, k := range keys {
+		nsl := reflect.New(sl.Type())
+		nsl.Elem().Set(reflect.Append(nsl.Elem(), m[k]...))
+		res = append(res, MakeSlice(nsl.Interface()))
+	}
+	return res
+}
+
+// SelectAnyOf filters s to contain only elements for which attr matches
+// any of the values.
+func (s Slice) SelectAnyOf(attr string, values ...string) {
+	index := s.indexForAttr(attr)
+	s.Filter(func(e Elem) bool {
+		vf := reflect.ValueOf(e).Elem().FieldByIndex(index)
+		return in(values, vf.String())
+	})
+}
+
+// SelectOnePerGroup filters s to include at most one element e per group of
+// elements matching Key(attr), where e has an attribute a that matches any
+// the values in v.
+// If more than one element in a group matches a value in v preference
+// is given to the element that matches the first value in v.
+func (s Slice) SelectOnePerGroup(a string, v []string) {
+	index := s.indexForAttr(a)
+	grouped := s.Group(func(e Elem) string { return Key(e, a) })
+	sl := s.Value()
+	sl.Set(sl.Slice(0, 0))
+	for _, g := range grouped {
+		e := reflect.Value{}
+		found := len(v)
+		gsl := g.Value()
+		for i := 0; i < gsl.Len(); i++ {
+			vi := gsl.Index(i).Elem().FieldByIndex(index)
+			j := 0
+			for ; j < len(v) && v[j] != vi.String(); j++ {
+			}
+			if j < found {
+				found = j
+				e = gsl.Index(i)
+			}
+		}
+		if found < len(v) {
+			sl.Set(reflect.Append(sl, e))
+		}
+	}
+}
+
+// SelectDraft drops all elements from the list with a draft level smaller than d
+// and selects the highest draft level of the remaining.
+// This method assumes that the input CLDR is canonicalized.
+func (s Slice) SelectDraft(d Draft) {
+	s.SelectOnePerGroup("draft", drafts[len(drafts)-2-int(d):])
+}

vendor/golang.org/x/text/unicode/norm/composition.go

@@ -0,0 +1,514 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+const (
+	maxNonStarters = 30
+	// The maximum number of characters needed for a buffer is
+	// maxNonStarters + 1 for the starter + 1 for the GCJ
+	maxBufferSize    = maxNonStarters + 2
+	maxNFCExpansion  = 3  // NFC(0x1D160)
+	maxNFKCExpansion = 18 // NFKC(0xFDFA)
+
+	maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128
+)
+
+// ssState is used for reporting the segment state after inserting a rune.
+// It is returned by streamSafe.next.
+type ssState int
+
+const (
+	// Indicates a rune was successfully added to the segment.
+	ssSuccess ssState = iota
+	// Indicates a rune starts a new segment and should not be added.
+	ssStarter
+	// Indicates a rune caused a segment overflow and a CGJ should be inserted.
+	ssOverflow
+)
+
+// streamSafe implements the policy of when a CGJ should be inserted.
+type streamSafe uint8
+
+// mkStreamSafe is a shorthand for declaring a streamSafe var and calling
+// first on it.
+func mkStreamSafe(p Properties) streamSafe {
+	return streamSafe(p.nTrailingNonStarters())
+}
+
+// first inserts the first rune of a segment.
+func (ss *streamSafe) first(p Properties) {
+	if *ss != 0 {
+		panic("!= 0")
+	}
+	*ss = streamSafe(p.nTrailingNonStarters())
+}
+
+// insert returns a ssState value to indicate whether a rune represented by p
+// can be inserted.
+func (ss *streamSafe) next(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	n := p.nLeadingNonStarters()
+	if *ss += streamSafe(n); *ss > maxNonStarters {
+		*ss = 0
+		return ssOverflow
+	}
+	// The Stream-Safe Text Processing prescribes that the counting can stop
+	// as soon as a starter is encountered. However, there are some starters,
+	// like Jamo V and T, that can combine with other runes, leaving their
+	// successive non-starters appended to the previous, possibly causing an
+	// overflow. We will therefore consider any rune with a non-zero nLead to
+	// be a non-starter. Note that it always hold that if nLead > 0 then
+	// nLead == nTrail.
+	if n == 0 {
+		*ss = 0
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+// backwards is used for checking for overflow and segment starts
+// when traversing a string backwards. Users do not need to call first
+// for the first rune. The state of the streamSafe retains the count of
+// the non-starters loaded.
+func (ss *streamSafe) backwards(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	c := *ss + streamSafe(p.nTrailingNonStarters())
+	if c > maxNonStarters {
+		return ssOverflow
+	}
+	*ss = c
+	if p.nLeadingNonStarters() == 0 {
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+func (ss streamSafe) isMax() bool {
+	return ss == maxNonStarters
+}
+
+// GraphemeJoiner is inserted after maxNonStarters non-starter runes.
+const GraphemeJoiner = "\u034F"
+
+// reorderBuffer is used to normalize a single segment.  Characters inserted with
+// insert are decomposed and reordered based on CCC. The compose method can
+// be used to recombine characters.  Note that the byte buffer does not hold
+// the UTF-8 characters in order.  Only the rune array is maintained in sorted
+// order. flush writes the resulting segment to a byte array.
+type reorderBuffer struct {
+	rune  [maxBufferSize]Properties // Per character info.
+	byte  [maxByteBufferSize]byte   // UTF-8 buffer. Referenced by runeInfo.pos.
+	nbyte uint8                     // Number or bytes.
+	ss    streamSafe                // For limiting length of non-starter sequence.
+	nrune int                       // Number of runeInfos.
+	f     formInfo
+
+	src      input
+	nsrc     int
+	tmpBytes input
+
+	out    []byte
+	flushF func(*reorderBuffer) bool
+}
+
+func (rb *reorderBuffer) init(f Form, src []byte) {
+	rb.f = *formTable[f]
+	rb.src.setBytes(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) initString(f Form, src string) {
+	rb.f = *formTable[f]
+	rb.src.setString(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
+	rb.out = out
+	rb.flushF = f
+}
+
+// reset discards all characters from the buffer.
+func (rb *reorderBuffer) reset() {
+	rb.nrune = 0
+	rb.nbyte = 0
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) doFlush() bool {
+	if rb.f.composing {
+		rb.compose()
+	}
+	res := rb.flushF(rb)
+	rb.reset()
+	return res
+}
+
+// appendFlush appends the normalized segment to rb.out.
+func appendFlush(rb *reorderBuffer) bool {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		rb.out = append(rb.out, rb.byte[start:end]...)
+	}
+	return true
+}
+
+// flush appends the normalized segment to out and resets rb.
+func (rb *reorderBuffer) flush(out []byte) []byte {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		out = append(out, rb.byte[start:end]...)
+	}
+	rb.reset()
+	return out
+}
+
+// flushCopy copies the normalized segment to buf and resets rb.
+// It returns the number of bytes written to buf.
+func (rb *reorderBuffer) flushCopy(buf []byte) int {
+	p := 0
+	for i := 0; i < rb.nrune; i++ {
+		runep := rb.rune[i]
+		p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size])
+	}
+	rb.reset()
+	return p
+}
+
+// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class.
+// It returns false if the buffer is not large enough to hold the rune.
+// It is used internally by insert and insertString only.
+func (rb *reorderBuffer) insertOrdered(info Properties) {
+	n := rb.nrune
+	b := rb.rune[:]
+	cc := info.ccc
+	if cc > 0 {
+		// Find insertion position + move elements to make room.
+		for ; n > 0; n-- {
+			if b[n-1].ccc <= cc {
+				break
+			}
+			b[n] = b[n-1]
+		}
+	}
+	rb.nrune += 1
+	pos := uint8(rb.nbyte)
+	rb.nbyte += utf8.UTFMax
+	info.pos = pos
+	b[n] = info
+}
+
+// insertErr is an error code returned by insert. Using this type instead
+// of error improves performance up to 20% for many of the benchmarks.
+type insertErr int
+
+const (
+	iSuccess insertErr = -iota
+	iShortDst
+	iShortSrc
+)
+
+// insertFlush inserts the given rune in the buffer ordered by CCC.
+// If a decomposition with multiple segments are encountered, they leading
+// ones are flushed.
+// It returns a non-zero error code if the rune was not inserted.
+func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+		return iSuccess
+	}
+	if info.hasDecomposition() {
+		return rb.insertDecomposed(info.Decomposition())
+	}
+	rb.insertSingle(src, i, info)
+	return iSuccess
+}
+
+// insertUnsafe inserts the given rune in the buffer ordered by CCC.
+// It is assumed there is sufficient space to hold the runes. It is the
+// responsibility of the caller to ensure this. This can be done by checking
+// the state returned by the streamSafe type.
+func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+	}
+	if info.hasDecomposition() {
+		// TODO: inline.
+		rb.insertDecomposed(info.Decomposition())
+	} else {
+		rb.insertSingle(src, i, info)
+	}
+}
+
+// insertDecomposed inserts an entry in to the reorderBuffer for each rune
+// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes.
+// It flushes the buffer on each new segment start.
+func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
+	rb.tmpBytes.setBytes(dcomp)
+	for i := 0; i < len(dcomp); {
+		info := rb.f.info(rb.tmpBytes, i)
+		if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {
+			return iShortDst
+		}
+		i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)])
+		rb.insertOrdered(info)
+	}
+	return iSuccess
+}
+
+// insertSingle inserts an entry in the reorderBuffer for the rune at
+// position i. info is the runeInfo for the rune at position i.
+func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) {
+	src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size))
+	rb.insertOrdered(info)
+}
+
+// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb.
+func (rb *reorderBuffer) insertCGJ() {
+	rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))})
+}
+
+// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
+func (rb *reorderBuffer) appendRune(r rune) {
+	bn := rb.nbyte
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.nbyte += utf8.UTFMax
+	rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)}
+	rb.nrune++
+}
+
+// assignRune sets a rune at position pos. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) assignRune(pos int, r rune) {
+	bn := rb.rune[pos].pos
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.rune[pos] = Properties{pos: bn, size: uint8(sz)}
+}
+
+// runeAt returns the rune at position n. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) runeAt(n int) rune {
+	inf := rb.rune[n]
+	r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size])
+	return r
+}
+
+// bytesAt returns the UTF-8 encoding of the rune at position n.
+// It is used for Hangul and recomposition.
+func (rb *reorderBuffer) bytesAt(n int) []byte {
+	inf := rb.rune[n]
+	return rb.byte[inf.pos : int(inf.pos)+int(inf.size)]
+}
+
+// For Hangul we combine algorithmically, instead of using tables.
+const (
+	hangulBase  = 0xAC00 // UTF-8(hangulBase) -> EA B0 80
+	hangulBase0 = 0xEA
+	hangulBase1 = 0xB0
+	hangulBase2 = 0x80
+
+	hangulEnd  = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4
+	hangulEnd0 = 0xED
+	hangulEnd1 = 0x9E
+	hangulEnd2 = 0xA4
+
+	jamoLBase  = 0x1100 // UTF-8(jamoLBase) -> E1 84 00
+	jamoLBase0 = 0xE1
+	jamoLBase1 = 0x84
+	jamoLEnd   = 0x1113
+	jamoVBase  = 0x1161
+	jamoVEnd   = 0x1176
+	jamoTBase  = 0x11A7
+	jamoTEnd   = 0x11C3
+
+	jamoTCount   = 28
+	jamoVCount   = 21
+	jamoVTCount  = 21 * 28
+	jamoLVTCount = 19 * 21 * 28
+)
+
+const hangulUTF8Size = 3
+
+func isHangul(b []byte) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+func isHangulString(b string) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+// Caller must ensure len(b) >= 2.
+func isJamoVT(b []byte) bool {
+	// True if (rune & 0xff00) == jamoLBase
+	return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1
+}
+
+func isHangulWithoutJamoT(b []byte) bool {
+	c, _ := utf8.DecodeRune(b)
+	c -= hangulBase
+	return c < jamoLVTCount && c%jamoTCount == 0
+}
+
+// decomposeHangul writes the decomposed Hangul to buf and returns the number
+// of bytes written.  len(buf) should be at least 9.
+func decomposeHangul(buf []byte, r rune) int {
+	const JamoUTF8Len = 3
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
+	utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
+	if x != 0 {
+		utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
+		return 3 * JamoUTF8Len
+	}
+	return 2 * JamoUTF8Len
+}
+
+// decomposeHangul algorithmically decomposes a Hangul rune into
+// its Jamo components.
+// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul.
+func (rb *reorderBuffer) decomposeHangul(r rune) {
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	rb.appendRune(jamoLBase + r/jamoVCount)
+	rb.appendRune(jamoVBase + r%jamoVCount)
+	if x != 0 {
+		rb.appendRune(jamoTBase + x)
+	}
+}
+
+// combineHangul algorithmically combines Jamo character components into Hangul.
+// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul.
+func (rb *reorderBuffer) combineHangul(s, i, k int) {
+	b := rb.rune[:]
+	bn := rb.nrune
+	for ; i < bn; i++ {
+		cccB := b[k-1].ccc
+		cccC := b[i].ccc
+		if cccB == 0 {
+			s = k - 1
+		}
+		if s != k-1 && cccB >= cccC {
+			// b[i] is blocked by greater-equal cccX below it
+			b[k] = b[i]
+			k++
+		} else {
+			l := rb.runeAt(s) // also used to compare to hangulBase
+			v := rb.runeAt(i) // also used to compare to jamoT
+			switch {
+			case jamoLBase <= l && l < jamoLEnd &&
+				jamoVBase <= v && v < jamoVEnd:
+				// 11xx plus 116x to LV
+				rb.assignRune(s, hangulBase+
+					(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount)
+			case hangulBase <= l && l < hangulEnd &&
+				jamoTBase < v && v < jamoTEnd &&
+				((l-hangulBase)%jamoTCount) == 0:
+				// ACxx plus 11Ax to LVT
+				rb.assignRune(s, l+v-jamoTBase)
+			default:
+				b[k] = b[i]
+				k++
+			}
+		}
+	}
+	rb.nrune = k
+}
+
+// compose recombines the runes in the buffer.
+// It should only be used to recompose a single segment, as it will not
+// handle alternations between Hangul and non-Hangul characters correctly.
+func (rb *reorderBuffer) compose() {
+	// UAX #15, section X5 , including Corrigendum #5
+	// "In any character sequence beginning with starter S, a character C is
+	//  blocked from S if and only if there is some character B between S
+	//  and C, and either B is a starter or it has the same or higher
+	//  combining class as C."
+	bn := rb.nrune
+	if bn == 0 {
+		return
+	}
+	k := 1
+	b := rb.rune[:]
+	for s, i := 0, 1; i < bn; i++ {
+		if isJamoVT(rb.bytesAt(i)) {
+			// Redo from start in Hangul mode. Necessary to support
+			// U+320E..U+321E in NFKC mode.
+			rb.combineHangul(s, i, k)
+			return
+		}
+		ii := b[i]
+		// We can only use combineForward as a filter if we later
+		// get the info for the combined character. This is more
+		// expensive than using the filter. Using combinesBackward()
+		// is safe.
+		if ii.combinesBackward() {
+			cccB := b[k-1].ccc
+			cccC := ii.ccc
+			blocked := false // b[i] blocked by starter or greater or equal CCC?
+			if cccB == 0 {
+				s = k - 1
+			} else {
+				blocked = s != k-1 && cccB >= cccC
+			}
+			if !blocked {
+				combined := combine(rb.runeAt(s), rb.runeAt(i))
+				if combined != 0 {
+					rb.assignRune(s, combined)
+					continue
+				}
+			}
+		}
+		b[k] = b[i]
+		k++
+	}
+	rb.nrune = k
+}

vendor/golang.org/x/text/unicode/norm/forminfo.go

@@ -0,0 +1,259 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+// This file contains Form-specific logic and wrappers for data in tables.go.
+
+// Rune info is stored in a separate trie per composing form. A composing form
+// and its corresponding decomposing form share the same trie.  Each trie maps
+// a rune to a uint16. The values take two forms.  For v >= 0x8000:
+//   bits
+//   15:    1 (inverse of NFD_QC bit of qcInfo)
+//   13..7: qcInfo (see below). isYesD is always true (no decompostion).
+//    6..0: ccc (compressed CCC value).
+// For v < 0x8000, the respective rune has a decomposition and v is an index
+// into a byte array of UTF-8 decomposition sequences and additional info and
+// has the form:
+//    <header> <decomp_byte>* [<tccc> [<lccc>]]
+// The header contains the number of bytes in the decomposition (excluding this
+// length byte). The two most significant bits of this length byte correspond
+// to bit 5 and 4 of qcInfo (see below).  The byte sequence itself starts at v+1.
+// The byte sequence is followed by a trailing and leading CCC if the values
+// for these are not zero.  The value of v determines which ccc are appended
+// to the sequences.  For v < firstCCC, there are none, for v >= firstCCC,
+// the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
+// there is an additional leading ccc. The value of tccc itself is the
+// trailing CCC shifted left 2 bits. The two least-significant bits of tccc
+// are the number of trailing non-starters.
+
+const (
+	qcInfoMask      = 0x3F // to clear all but the relevant bits in a qcInfo
+	headerLenMask   = 0x3F // extract the length value from the header byte
+	headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
+)
+
+// Properties provides access to normalization properties of a rune.
+type Properties struct {
+	pos   uint8  // start position in reorderBuffer; used in composition.go
+	size  uint8  // length of UTF-8 encoding of this rune
+	ccc   uint8  // leading canonical combining class (ccc if not decomposition)
+	tccc  uint8  // trailing canonical combining class (ccc if not decomposition)
+	nLead uint8  // number of leading non-starters.
+	flags qcInfo // quick check flags
+	index uint16
+}
+
+// functions dispatchable per form
+type lookupFunc func(b input, i int) Properties
+
+// formInfo holds Form-specific functions and tables.
+type formInfo struct {
+	form                     Form
+	composing, compatibility bool // form type
+	info                     lookupFunc
+	nextMain                 iterFunc
+}
+
+var formTable = []*formInfo{{
+	form:          NFC,
+	composing:     true,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFD,
+	composing:     false,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextDecomposed,
+}, {
+	form:          NFKC,
+	composing:     true,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFKD,
+	composing:     false,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextDecomposed,
+}}
+
+// We do not distinguish between boundaries for NFC, NFD, etc. to avoid
+// unexpected behavior for the user.  For example, in NFD, there is a boundary
+// after 'a'.  However, 'a' might combine with modifiers, so from the application's
+// perspective it is not a good boundary. We will therefore always use the
+// boundaries for the combining variants.
+
+// BoundaryBefore returns true if this rune starts a new segment and
+// cannot combine with any rune on the left.
+func (p Properties) BoundaryBefore() bool {
+	if p.ccc == 0 && !p.combinesBackward() {
+		return true
+	}
+	// We assume that the CCC of the first character in a decomposition
+	// is always non-zero if different from info.ccc and that we can return
+	// false at this point. This is verified by maketables.
+	return false
+}
+
+// BoundaryAfter returns true if runes cannot combine with or otherwise
+// interact with this or previous runes.
+func (p Properties) BoundaryAfter() bool {
+	// TODO: loosen these conditions.
+	return p.isInert()
+}
+
+// We pack quick check data in 4 bits:
+//   5:    Combines forward  (0 == false, 1 == true)
+//   4..3: NFC_QC Yes(00), No (10), or Maybe (11)
+//   2:    NFD_QC Yes (0) or No (1). No also means there is a decomposition.
+//   1..0: Number of trailing non-starters.
+//
+// When all 4 bits are zero, the character is inert, meaning it is never
+// influenced by normalization.
+type qcInfo uint8
+
+func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
+func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
+
+func (p Properties) combinesForward() bool  { return p.flags&0x20 != 0 }
+func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
+func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
+
+func (p Properties) isInert() bool {
+	return p.flags&qcInfoMask == 0 && p.ccc == 0
+}
+
+func (p Properties) multiSegment() bool {
+	return p.index >= firstMulti && p.index < endMulti
+}
+
+func (p Properties) nLeadingNonStarters() uint8 {
+	return p.nLead
+}
+
+func (p Properties) nTrailingNonStarters() uint8 {
+	return uint8(p.flags & 0x03)
+}
+
+// Decomposition returns the decomposition for the underlying rune
+// or nil if there is none.
+func (p Properties) Decomposition() []byte {
+	// TODO: create the decomposition for Hangul?
+	if p.index == 0 {
+		return nil
+	}
+	i := p.index
+	n := decomps[i] & headerLenMask
+	i++
+	return decomps[i : i+uint16(n)]
+}
+
+// Size returns the length of UTF-8 encoding of the rune.
+func (p Properties) Size() int {
+	return int(p.size)
+}
+
+// CCC returns the canonical combining class of the underlying rune.
+func (p Properties) CCC() uint8 {
+	if p.index >= firstCCCZeroExcept {
+		return 0
+	}
+	return ccc[p.ccc]
+}
+
+// LeadCCC returns the CCC of the first rune in the decomposition.
+// If there is no decomposition, LeadCCC equals CCC.
+func (p Properties) LeadCCC() uint8 {
+	return ccc[p.ccc]
+}
+
+// TrailCCC returns the CCC of the last rune in the decomposition.
+// If there is no decomposition, TrailCCC equals CCC.
+func (p Properties) TrailCCC() uint8 {
+	return ccc[p.tccc]
+}
+
+// Recomposition
+// We use 32-bit keys instead of 64-bit for the two codepoint keys.
+// This clips off the bits of three entries, but we know this will not
+// result in a collision. In the unlikely event that changes to
+// UnicodeData.txt introduce collisions, the compiler will catch it.
+// Note that the recomposition map for NFC and NFKC are identical.
+
+// combine returns the combined rune or 0 if it doesn't exist.
+func combine(a, b rune) rune {
+	key := uint32(uint16(a))<<16 + uint32(uint16(b))
+	return recompMap[key]
+}
+
+func lookupInfoNFC(b input, i int) Properties {
+	v, sz := b.charinfoNFC(i)
+	return compInfo(v, sz)
+}
+
+func lookupInfoNFKC(b input, i int) Properties {
+	v, sz := b.charinfoNFKC(i)
+	return compInfo(v, sz)
+}
+
+// Properties returns properties for the first rune in s.
+func (f Form) Properties(s []byte) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookup(s))
+	}
+	return compInfo(nfkcData.lookup(s))
+}
+
+// PropertiesString returns properties for the first rune in s.
+func (f Form) PropertiesString(s string) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookupString(s))
+	}
+	return compInfo(nfkcData.lookupString(s))
+}
+
+// compInfo converts the information contained in v and sz
+// to a Properties.  See the comment at the top of the file
+// for more information on the format.
+func compInfo(v uint16, sz int) Properties {
+	if v == 0 {
+		return Properties{size: uint8(sz)}
+	} else if v >= 0x8000 {
+		p := Properties{
+			size:  uint8(sz),
+			ccc:   uint8(v),
+			tccc:  uint8(v),
+			flags: qcInfo(v >> 8),
+		}
+		if p.ccc > 0 || p.combinesBackward() {
+			p.nLead = uint8(p.flags & 0x3)
+		}
+		return p
+	}
+	// has decomposition
+	h := decomps[v]
+	f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
+	p := Properties{size: uint8(sz), flags: f, index: v}
+	if v >= firstCCC {
+		v += uint16(h&headerLenMask) + 1
+		c := decomps[v]
+		p.tccc = c >> 2
+		p.flags |= qcInfo(c & 0x3)
+		if v >= firstLeadingCCC {
+			p.nLead = c & 0x3
+			if v >= firstStarterWithNLead {
+				// We were tricked. Remove the decomposition.
+				p.flags &= 0x03
+				p.index = 0
+				return p
+			}
+			p.ccc = decomps[v+1]
+		}
+	}
+	return p
+}

vendor/golang.org/x/text/unicode/norm/input.go

@@ -0,0 +1,105 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+type input struct {
+	str   string
+	bytes []byte
+}
+
+func inputBytes(str []byte) input {
+	return input{bytes: str}
+}
+
+func inputString(str string) input {
+	return input{str: str}
+}
+
+func (in *input) setBytes(str []byte) {
+	in.str = ""
+	in.bytes = str
+}
+
+func (in *input) setString(str string) {
+	in.str = str
+	in.bytes = nil
+}
+
+func (in *input) _byte(p int) byte {
+	if in.bytes == nil {
+		return in.str[p]
+	}
+	return in.bytes[p]
+}
+
+func (in *input) skipASCII(p, max int) int {
+	if in.bytes == nil {
+		for ; p < max && in.str[p] < utf8.RuneSelf; p++ {
+		}
+	} else {
+		for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) skipContinuationBytes(p int) int {
+	if in.bytes == nil {
+		for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ {
+		}
+	} else {
+		for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) appendSlice(buf []byte, b, e int) []byte {
+	if in.bytes != nil {
+		return append(buf, in.bytes[b:e]...)
+	}
+	for i := b; i < e; i++ {
+		buf = append(buf, in.str[i])
+	}
+	return buf
+}
+
+func (in *input) copySlice(buf []byte, b, e int) int {
+	if in.bytes == nil {
+		return copy(buf, in.str[b:e])
+	}
+	return copy(buf, in.bytes[b:e])
+}
+
+func (in *input) charinfoNFC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfcData.lookupString(in.str[p:])
+	}
+	return nfcData.lookup(in.bytes[p:])
+}
+
+func (in *input) charinfoNFKC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfkcData.lookupString(in.str[p:])
+	}
+	return nfkcData.lookup(in.bytes[p:])
+}
+
+func (in *input) hangul(p int) (r rune) {
+	if in.bytes == nil {
+		if !isHangulString(in.str[p:]) {
+			return 0
+		}
+		r, _ = utf8.DecodeRuneInString(in.str[p:])
+	} else {
+		if !isHangul(in.bytes[p:]) {
+			return 0
+		}
+		r, _ = utf8.DecodeRune(in.bytes[p:])
+	}
+	return r
+}

vendor/golang.org/x/text/unicode/norm/iter.go

@@ -0,0 +1,450 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"fmt"
+	"unicode/utf8"
+)
+
+// MaxSegmentSize is the maximum size of a byte buffer needed to consider any
+// sequence of starter and non-starter runes for the purpose of normalization.
+const MaxSegmentSize = maxByteBufferSize
+
+// An Iter iterates over a string or byte slice, while normalizing it
+// to a given Form.
+type Iter struct {
+	rb     reorderBuffer
+	buf    [maxByteBufferSize]byte
+	info   Properties // first character saved from previous iteration
+	next   iterFunc   // implementation of next depends on form
+	asciiF iterFunc
+
+	p        int    // current position in input source
+	multiSeg []byte // remainder of multi-segment decomposition
+}
+
+type iterFunc func(*Iter) []byte
+
+// Init initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) Init(f Form, src []byte) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.init(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIBytes
+	i.info = i.rb.f.info(i.rb.src, i.p)
+}
+
+// InitString initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) InitString(f Form, src string) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.initString(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIString
+	i.info = i.rb.f.info(i.rb.src, i.p)
+}
+
+// Seek sets the segment to be returned by the next call to Next to start
+// at position p.  It is the responsibility of the caller to set p to the
+// start of a UTF8 rune.
+func (i *Iter) Seek(offset int64, whence int) (int64, error) {
+	var abs int64
+	switch whence {
+	case 0:
+		abs = offset
+	case 1:
+		abs = int64(i.p) + offset
+	case 2:
+		abs = int64(i.rb.nsrc) + offset
+	default:
+		return 0, fmt.Errorf("norm: invalid whence")
+	}
+	if abs < 0 {
+		return 0, fmt.Errorf("norm: negative position")
+	}
+	if int(abs) >= i.rb.nsrc {
+		i.setDone()
+		return int64(i.p), nil
+	}
+	i.p = int(abs)
+	i.multiSeg = nil
+	i.next = i.rb.f.nextMain
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	return abs, nil
+}
+
+// returnSlice returns a slice of the underlying input type as a byte slice.
+// If the underlying is of type []byte, it will simply return a slice.
+// If the underlying is of type string, it will copy the slice to the buffer
+// and return that.
+func (i *Iter) returnSlice(a, b int) []byte {
+	if i.rb.src.bytes == nil {
+		return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
+	}
+	return i.rb.src.bytes[a:b]
+}
+
+// Pos returns the byte position at which the next call to Next will commence processing.
+func (i *Iter) Pos() int {
+	return i.p
+}
+
+func (i *Iter) setDone() {
+	i.next = nextDone
+	i.p = i.rb.nsrc
+}
+
+// Done returns true if there is no more input to process.
+func (i *Iter) Done() bool {
+	return i.p >= i.rb.nsrc
+}
+
+// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
+// For any input a and b for which f(a) == f(b), subsequent calls
+// to Next will return the same segments.
+// Modifying runes are grouped together with the preceding starter, if such a starter exists.
+// Although not guaranteed, n will typically be the smallest possible n.
+func (i *Iter) Next() []byte {
+	return i.next(i)
+}
+
+func nextASCIIBytes(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		i.setDone()
+		return i.rb.src.bytes[i.p:p]
+	}
+	if i.rb.src.bytes[p] < utf8.RuneSelf {
+		p0 := i.p
+		i.p = p
+		return i.rb.src.bytes[p0:p]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextASCIIString(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.setDone()
+		return i.buf[:1]
+	}
+	if i.rb.src.str[p] < utf8.RuneSelf {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.p = p
+		return i.buf[:1]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextHangul(i *Iter) []byte {
+	p := i.p
+	next := p + hangulUTF8Size
+	if next >= i.rb.nsrc {
+		i.setDone()
+	} else if i.rb.src.hangul(next) == 0 {
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		i.next = i.rb.f.nextMain
+		return i.next(i)
+	}
+	i.p = next
+	return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
+}
+
+func nextDone(i *Iter) []byte {
+	return nil
+}
+
+// nextMulti is used for iterating over multi-segment decompositions
+// for decomposing normal forms.
+func nextMulti(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	// skip first rune
+	for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
+	}
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.multiSeg = d[j:]
+			return d[:j]
+		}
+		j += int(info.size)
+	}
+	// treat last segment as normal decomposition
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+// nextMultiNorm is used for iterating over multi-segment decompositions
+// for composing normal forms.
+func nextMultiNorm(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.rb.compose()
+			seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+			i.rb.ss.first(info)
+			i.rb.insertUnsafe(input{bytes: d}, j, info)
+			i.multiSeg = d[j+int(info.size):]
+			return seg
+		}
+		i.rb.ss.next(info)
+		i.rb.insertUnsafe(input{bytes: d}, j, info)
+		j += int(info.size)
+	}
+	i.multiSeg = nil
+	i.next = nextComposed
+	return doNormComposed(i)
+}
+
+// nextDecomposed is the implementation of Next for forms NFD and NFKD.
+func nextDecomposed(i *Iter) (next []byte) {
+	outp := 0
+	inCopyStart, outCopyStart := i.p, 0
+	ss := mkStreamSafe(i.info)
+	for {
+		if sz := int(i.info.size); sz <= 1 {
+			p := i.p
+			i.p++ // ASCII or illegal byte.  Either way, advance by 1.
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				return i.returnSlice(p, i.p)
+			} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+				i.next = i.asciiF
+				return i.returnSlice(p, i.p)
+			}
+			outp++
+		} else if d := i.info.Decomposition(); d != nil {
+			// Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
+			// Case 1: there is a leftover to copy.  In this case the decomposition
+			// must begin with a modifier and should always be appended.
+			// Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
+			p := outp + len(d)
+			if outp > 0 {
+				i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+				if p > len(i.buf) {
+					return i.buf[:outp]
+				}
+			} else if i.info.multiSegment() {
+				// outp must be 0 as multi-segment decompositions always
+				// start a new segment.
+				if i.multiSeg == nil {
+					i.multiSeg = d
+					i.next = nextMulti
+					return nextMulti(i)
+				}
+				// We are in the last segment.  Treat as normal decomposition.
+				d = i.multiSeg
+				i.multiSeg = nil
+				p = len(d)
+			}
+			prevCC := i.info.tccc
+			if i.p += sz; i.p >= i.rb.nsrc {
+				i.setDone()
+				i.info = Properties{} // Force BoundaryBefore to succeed.
+			} else {
+				i.info = i.rb.f.info(i.rb.src, i.p)
+			}
+			switch ss.next(i.info) {
+			case ssOverflow:
+				i.next = nextCGJDecompose
+				fallthrough
+			case ssStarter:
+				if outp > 0 {
+					copy(i.buf[outp:], d)
+					return i.buf[:p]
+				}
+				return d
+			}
+			copy(i.buf[outp:], d)
+			outp = p
+			inCopyStart, outCopyStart = i.p, outp
+			if i.info.ccc < prevCC {
+				goto doNorm
+			}
+			continue
+		} else if r := i.rb.src.hangul(i.p); r != 0 {
+			outp = decomposeHangul(i.buf[:], r)
+			i.p += hangulUTF8Size
+			inCopyStart, outCopyStart = i.p, outp
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				break
+			} else if i.rb.src.hangul(i.p) != 0 {
+				i.next = nextHangul
+				return i.buf[:outp]
+			}
+		} else {
+			p := outp + sz
+			if p > len(i.buf) {
+				break
+			}
+			outp = p
+			i.p += sz
+		}
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		prevCC := i.info.tccc
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	if outCopyStart == 0 {
+		return i.returnSlice(inCopyStart, i.p)
+	} else if inCopyStart < i.p {
+		i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	}
+	return i.buf[:outp]
+doNorm:
+	// Insert what we have decomposed so far in the reorderBuffer.
+	// As we will only reorder, there will always be enough room.
+	i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	i.rb.insertDecomposed(i.buf[0:outp])
+	return doNormDecomposed(i)
+}
+
+func doNormDecomposed(i *Iter) []byte {
+	for {
+		if s := i.rb.ss.next(i.info); s == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if i.info.ccc == 0 {
+			break
+		}
+	}
+	// new segment or too many combining characters: exit normalization
+	return i.buf[:i.rb.flushCopy(i.buf[:])]
+}
+
+func nextCGJDecompose(i *Iter) []byte {
+	i.rb.ss = 0
+	i.rb.insertCGJ()
+	i.next = nextDecomposed
+	buf := doNormDecomposed(i)
+	return buf
+}
+
+// nextComposed is the implementation of Next for forms NFC and NFKC.
+func nextComposed(i *Iter) []byte {
+	outp, startp := 0, i.p
+	var prevCC uint8
+	ss := mkStreamSafe(i.info)
+	for {
+		if !i.info.isYesC() {
+			goto doNorm
+		}
+		prevCC = i.info.tccc
+		sz := int(i.info.size)
+		if sz == 0 {
+			sz = 1 // illegal rune: copy byte-by-byte
+		}
+		p := outp + sz
+		if p > len(i.buf) {
+			break
+		}
+		outp = p
+		i.p += sz
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+			i.next = i.asciiF
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	return i.returnSlice(startp, i.p)
+doNorm:
+	i.p = startp
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	if i.info.multiSegment() {
+		d := i.info.Decomposition()
+		info := i.rb.f.info(input{bytes: d}, 0)
+		i.rb.insertUnsafe(input{bytes: d}, 0, info)
+		i.multiSeg = d[int(info.size):]
+		i.next = nextMultiNorm
+		return nextMultiNorm(i)
+	}
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}
+
+func doNormComposed(i *Iter) []byte {
+	// First rune should already be inserted.
+	for {
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if s := i.rb.ss.next(i.info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	}
+	i.rb.compose()
+	seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+	return seg
+}
+
+func nextCGJCompose(i *Iter) []byte {
+	i.rb.ss = 0 // instead of first
+	i.rb.insertCGJ()
+	i.next = nextComposed
+	// Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
+	// even if they are not. This is particularly dubious for U+FF9E and UFF9A.
+	// If we ever change that, insert a check here.
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}

vendor/golang.org/x/text/unicode/norm/maketables.go

@@ -0,0 +1,976 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Normalization table generator.
+// Data read from the web.
+// See forminfo.go for a description of the trie values associated with each rune.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/triegen"
+	"golang.org/x/text/internal/ucd"
+)
+
+func main() {
+	gen.Init()
+	loadUnicodeData()
+	compactCCC()
+	loadCompositionExclusions()
+	completeCharFields(FCanonical)
+	completeCharFields(FCompatibility)
+	computeNonStarterCounts()
+	verifyComputed()
+	printChars()
+	testDerived()
+	printTestdata()
+	makeTables()
+}
+
+var (
+	tablelist = flag.String("tables",
+		"all",
+		"comma-separated list of which tables to generate; "+
+			"can be 'decomp', 'recomp', 'info' and 'all'")
+	test = flag.Bool("test",
+		false,
+		"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
+	verbose = flag.Bool("verbose",
+		false,
+		"write data to stdout as it is parsed")
+)
+
+const MaxChar = 0x10FFFF // anything above this shouldn't exist
+
+// Quick Check properties of runes allow us to quickly
+// determine whether a rune may occur in a normal form.
+// For a given normal form, a rune may be guaranteed to occur
+// verbatim (QC=Yes), may or may not combine with another
+// rune (QC=Maybe), or may not occur (QC=No).
+type QCResult int
+
+const (
+	QCUnknown QCResult = iota
+	QCYes
+	QCNo
+	QCMaybe
+)
+
+func (r QCResult) String() string {
+	switch r {
+	case QCYes:
+		return "Yes"
+	case QCNo:
+		return "No"
+	case QCMaybe:
+		return "Maybe"
+	}
+	return "***UNKNOWN***"
+}
+
+const (
+	FCanonical     = iota // NFC or NFD
+	FCompatibility        // NFKC or NFKD
+	FNumberOfFormTypes
+)
+
+const (
+	MComposed   = iota // NFC or NFKC
+	MDecomposed        // NFD or NFKD
+	MNumberOfModes
+)
+
+// This contains only the properties we're interested in.
+type Char struct {
+	name          string
+	codePoint     rune  // if zero, this index is not a valid code point.
+	ccc           uint8 // canonical combining class
+	origCCC       uint8
+	excludeInComp bool // from CompositionExclusions.txt
+	compatDecomp  bool // it has a compatibility expansion
+
+	nTrailingNonStarters uint8
+	nLeadingNonStarters  uint8 // must be equal to trailing if non-zero
+
+	forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility
+
+	state State
+}
+
+var chars = make([]Char, MaxChar+1)
+var cccMap = make(map[uint8]uint8)
+
+func (c Char) String() string {
+	buf := new(bytes.Buffer)
+
+	fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name)
+	fmt.Fprintf(buf, "  ccc: %v\n", c.ccc)
+	fmt.Fprintf(buf, "  excludeInComp: %v\n", c.excludeInComp)
+	fmt.Fprintf(buf, "  compatDecomp: %v\n", c.compatDecomp)
+	fmt.Fprintf(buf, "  state: %v\n", c.state)
+	fmt.Fprintf(buf, "  NFC:\n")
+	fmt.Fprint(buf, c.forms[FCanonical])
+	fmt.Fprintf(buf, "  NFKC:\n")
+	fmt.Fprint(buf, c.forms[FCompatibility])
+
+	return buf.String()
+}
+
+// In UnicodeData.txt, some ranges are marked like this:
+//	3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
+//	4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
+// parseCharacter keeps a state variable indicating the weirdness.
+type State int
+
+const (
+	SNormal State = iota // known to be zero for the type
+	SFirst
+	SLast
+	SMissing
+)
+
+var lastChar = rune('\u0000')
+
+func (c Char) isValid() bool {
+	return c.codePoint != 0 && c.state != SMissing
+}
+
+type FormInfo struct {
+	quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed
+	verified   [MNumberOfModes]bool     // index: MComposed or MDecomposed
+
+	combinesForward  bool // May combine with rune on the right
+	combinesBackward bool // May combine with rune on the left
+	isOneWay         bool // Never appears in result
+	inDecomp         bool // Some decompositions result in this char.
+	decomp           Decomposition
+	expandedDecomp   Decomposition
+}
+
+func (f FormInfo) String() string {
+	buf := bytes.NewBuffer(make([]byte, 0))
+
+	fmt.Fprintf(buf, "    quickCheck[C]: %v\n", f.quickCheck[MComposed])
+	fmt.Fprintf(buf, "    quickCheck[D]: %v\n", f.quickCheck[MDecomposed])
+	fmt.Fprintf(buf, "    cmbForward: %v\n", f.combinesForward)
+	fmt.Fprintf(buf, "    cmbBackward: %v\n", f.combinesBackward)
+	fmt.Fprintf(buf, "    isOneWay: %v\n", f.isOneWay)
+	fmt.Fprintf(buf, "    inDecomp: %v\n", f.inDecomp)
+	fmt.Fprintf(buf, "    decomposition: %X\n", f.decomp)
+	fmt.Fprintf(buf, "    expandedDecomp: %X\n", f.expandedDecomp)
+
+	return buf.String()
+}
+
+type Decomposition []rune
+
+func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
+	decomp := strings.Split(s, " ")
+	if len(decomp) > 0 && skipfirst {
+		decomp = decomp[1:]
+	}
+	for _, d := range decomp {
+		point, err := strconv.ParseUint(d, 16, 64)
+		if err != nil {
+			return a, err
+		}
+		a = append(a, rune(point))
+	}
+	return a, nil
+}
+
+func loadUnicodeData() {
+	f := gen.OpenUCDFile("UnicodeData.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(ucd.CodePoint)
+		char := &chars[r]
+
+		char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass))
+		decmap := p.String(ucd.DecompMapping)
+
+		exp, err := parseDecomposition(decmap, false)
+		isCompat := false
+		if err != nil {
+			if len(decmap) > 0 {
+				exp, err = parseDecomposition(decmap, true)
+				if err != nil {
+					log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
+				}
+				isCompat = true
+			}
+		}
+
+		char.name = p.String(ucd.Name)
+		char.codePoint = r
+		char.forms[FCompatibility].decomp = exp
+		if !isCompat {
+			char.forms[FCanonical].decomp = exp
+		} else {
+			char.compatDecomp = true
+		}
+		if len(decmap) > 0 {
+			char.forms[FCompatibility].decomp = exp
+		}
+	}
+	if err := p.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// compactCCC converts the sparse set of CCC values to a continguous one,
+// reducing the number of bits needed from 8 to 6.
+func compactCCC() {
+	m := make(map[uint8]uint8)
+	for i := range chars {
+		c := &chars[i]
+		m[c.ccc] = 0
+	}
+	cccs := []int{}
+	for v, _ := range m {
+		cccs = append(cccs, int(v))
+	}
+	sort.Ints(cccs)
+	for i, c := range cccs {
+		cccMap[uint8(i)] = uint8(c)
+		m[uint8(c)] = uint8(i)
+	}
+	for i := range chars {
+		c := &chars[i]
+		c.origCCC = c.ccc
+		c.ccc = m[c.ccc]
+	}
+	if len(m) >= 1<<6 {
+		log.Fatalf("too many difference CCC values: %d >= 64", len(m))
+	}
+}
+
+// CompositionExclusions.txt has form:
+// 0958    # ...
+// See http://unicode.org/reports/tr44/ for full explanation
+func loadCompositionExclusions() {
+	f := gen.OpenUCDFile("CompositionExclusions.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		c := &chars[p.Rune(0)]
+		if c.excludeInComp {
+			log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
+		}
+		c.excludeInComp = true
+	}
+	if e := p.Err(); e != nil {
+		log.Fatal(e)
+	}
+}
+
+// hasCompatDecomp returns true if any of the recursive
+// decompositions contains a compatibility expansion.
+// In this case, the character may not occur in NFK*.
+func hasCompatDecomp(r rune) bool {
+	c := &chars[r]
+	if c.compatDecomp {
+		return true
+	}
+	for _, d := range c.forms[FCompatibility].decomp {
+		if hasCompatDecomp(d) {
+			return true
+		}
+	}
+	return false
+}
+
+// Hangul related constants.
+const (
+	HangulBase = 0xAC00
+	HangulEnd  = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28)
+
+	JamoLBase = 0x1100
+	JamoLEnd  = 0x1113
+	JamoVBase = 0x1161
+	JamoVEnd  = 0x1176
+	JamoTBase = 0x11A8
+	JamoTEnd  = 0x11C3
+
+	JamoLVTCount = 19 * 21 * 28
+	JamoTCount   = 28
+)
+
+func isHangul(r rune) bool {
+	return HangulBase <= r && r < HangulEnd
+}
+
+func isHangulWithoutJamoT(r rune) bool {
+	if !isHangul(r) {
+		return false
+	}
+	r -= HangulBase
+	return r < JamoLVTCount && r%JamoTCount == 0
+}
+
+func ccc(r rune) uint8 {
+	return chars[r].ccc
+}
+
+// Insert a rune in a buffer, ordered by Canonical Combining Class.
+func insertOrdered(b Decomposition, r rune) Decomposition {
+	n := len(b)
+	b = append(b, 0)
+	cc := ccc(r)
+	if cc > 0 {
+		// Use bubble sort.
+		for ; n > 0; n-- {
+			if ccc(b[n-1]) <= cc {
+				break
+			}
+			b[n] = b[n-1]
+		}
+	}
+	b[n] = r
+	return b
+}
+
+// Recursively decompose.
+func decomposeRecursive(form int, r rune, d Decomposition) Decomposition {
+	dcomp := chars[r].forms[form].decomp
+	if len(dcomp) == 0 {
+		return insertOrdered(d, r)
+	}
+	for _, c := range dcomp {
+		d = decomposeRecursive(form, c, d)
+	}
+	return d
+}
+
+func completeCharFields(form int) {
+	// Phase 0: pre-expand decomposition.
+	for i := range chars {
+		f := &chars[i].forms[form]
+		if len(f.decomp) == 0 {
+			continue
+		}
+		exp := make(Decomposition, 0)
+		for _, c := range f.decomp {
+			exp = decomposeRecursive(form, c, exp)
+		}
+		f.expandedDecomp = exp
+	}
+
+	// Phase 1: composition exclusion, mark decomposition.
+	for i := range chars {
+		c := &chars[i]
+		f := &c.forms[form]
+
+		// Marks script-specific exclusions and version restricted.
+		f.isOneWay = c.excludeInComp
+
+		// Singletons
+		f.isOneWay = f.isOneWay || len(f.decomp) == 1
+
+		// Non-starter decompositions
+		if len(f.decomp) > 1 {
+			chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0
+			f.isOneWay = f.isOneWay || chk
+		}
+
+		// Runes that decompose into more than two runes.
+		f.isOneWay = f.isOneWay || len(f.decomp) > 2
+
+		if form == FCompatibility {
+			f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint)
+		}
+
+		for _, r := range f.decomp {
+			chars[r].forms[form].inDecomp = true
+		}
+	}
+
+	// Phase 2: forward and backward combining.
+	for i := range chars {
+		c := &chars[i]
+		f := &c.forms[form]
+
+		if !f.isOneWay && len(f.decomp) == 2 {
+			f0 := &chars[f.decomp[0]].forms[form]
+			f1 := &chars[f.decomp[1]].forms[form]
+			if !f0.isOneWay {
+				f0.combinesForward = true
+			}
+			if !f1.isOneWay {
+				f1.combinesBackward = true
+			}
+		}
+		if isHangulWithoutJamoT(rune(i)) {
+			f.combinesForward = true
+		}
+	}
+
+	// Phase 3: quick check values.
+	for i := range chars {
+		c := &chars[i]
+		f := &c.forms[form]
+
+		switch {
+		case len(f.decomp) > 0:
+			f.quickCheck[MDecomposed] = QCNo
+		case isHangul(rune(i)):
+			f.quickCheck[MDecomposed] = QCNo
+		default:
+			f.quickCheck[MDecomposed] = QCYes
+		}
+		switch {
+		case f.isOneWay:
+			f.quickCheck[MComposed] = QCNo
+		case (i & 0xffff00) == JamoLBase:
+			f.quickCheck[MComposed] = QCYes
+			if JamoLBase <= i && i < JamoLEnd {
+				f.combinesForward = true
+			}
+			if JamoVBase <= i && i < JamoVEnd {
+				f.quickCheck[MComposed] = QCMaybe
+				f.combinesBackward = true
+				f.combinesForward = true
+			}
+			if JamoTBase <= i && i < JamoTEnd {
+				f.quickCheck[MComposed] = QCMaybe
+				f.combinesBackward = true
+			}
+		case !f.combinesBackward:
+			f.quickCheck[MComposed] = QCYes
+		default:
+			f.quickCheck[MComposed] = QCMaybe
+		}
+	}
+}
+
+func computeNonStarterCounts() {
+	// Phase 4: leading and trailing non-starter count
+	for i := range chars {
+		c := &chars[i]
+
+		runes := []rune{rune(i)}
+		// We always use FCompatibility so that the CGJ insertion points do not
+		// change for repeated normalizations with different forms.
+		if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 {
+			runes = exp
+		}
+		// We consider runes that combine backwards to be non-starters for the
+		// purpose of Stream-Safe Text Processing.
+		for _, r := range runes {
+			if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
+				break
+			}
+			c.nLeadingNonStarters++
+		}
+		for i := len(runes) - 1; i >= 0; i-- {
+			if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
+				break
+			}
+			c.nTrailingNonStarters++
+		}
+		if c.nTrailingNonStarters > 3 {
+			log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes)
+		}
+
+		if isHangul(rune(i)) {
+			c.nTrailingNonStarters = 2
+			if isHangulWithoutJamoT(rune(i)) {
+				c.nTrailingNonStarters = 1
+			}
+		}
+
+		if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t {
+			log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t)
+		}
+		if t := c.nTrailingNonStarters; t > 3 {
+			log.Fatalf("%U: number of trailing non-starters is %d > 3", t)
+		}
+	}
+}
+
+func printBytes(w io.Writer, b []byte, name string) {
+	fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
+	fmt.Fprintf(w, "var %s = [...]byte {", name)
+	for i, c := range b {
+		switch {
+		case i%64 == 0:
+			fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
+		case i%8 == 0:
+			fmt.Fprintf(w, "\n")
+		}
+		fmt.Fprintf(w, "0x%.2X, ", c)
+	}
+	fmt.Fprint(w, "\n}\n\n")
+}
+
+// See forminfo.go for format.
+func makeEntry(f *FormInfo, c *Char) uint16 {
+	e := uint16(0)
+	if r := c.codePoint; HangulBase <= r && r < HangulEnd {
+		e |= 0x40
+	}
+	if f.combinesForward {
+		e |= 0x20
+	}
+	if f.quickCheck[MDecomposed] == QCNo {
+		e |= 0x4
+	}
+	switch f.quickCheck[MComposed] {
+	case QCYes:
+	case QCNo:
+		e |= 0x10
+	case QCMaybe:
+		e |= 0x18
+	default:
+		log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed])
+	}
+	e |= uint16(c.nTrailingNonStarters)
+	return e
+}
+
+// decompSet keeps track of unique decompositions, grouped by whether
+// the decomposition is followed by a trailing and/or leading CCC.
+type decompSet [7]map[string]bool
+
+const (
+	normalDecomp = iota
+	firstMulti
+	firstCCC
+	endMulti
+	firstLeadingCCC
+	firstCCCZeroExcept
+	firstStarterWithNLead
+	lastDecomp
+)
+
+var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"}
+
+func makeDecompSet() decompSet {
+	m := decompSet{}
+	for i := range m {
+		m[i] = make(map[string]bool)
+	}
+	return m
+}
+func (m *decompSet) insert(key int, s string) {
+	m[key][s] = true
+}
+
+func printCharInfoTables(w io.Writer) int {
+	mkstr := func(r rune, f *FormInfo) (int, string) {
+		d := f.expandedDecomp
+		s := string([]rune(d))
+		if max := 1 << 6; len(s) >= max {
+			const msg = "%U: too many bytes in decomposition: %d >= %d"
+			log.Fatalf(msg, r, len(s), max)
+		}
+		head := uint8(len(s))
+		if f.quickCheck[MComposed] != QCYes {
+			head |= 0x40
+		}
+		if f.combinesForward {
+			head |= 0x80
+		}
+		s = string([]byte{head}) + s
+
+		lccc := ccc(d[0])
+		tccc := ccc(d[len(d)-1])
+		cc := ccc(r)
+		if cc != 0 && lccc == 0 && tccc == 0 {
+			log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
+		}
+		if tccc < lccc && lccc != 0 {
+			const msg = "%U: lccc (%d) must be <= tcc (%d)"
+			log.Fatalf(msg, r, lccc, tccc)
+		}
+		index := normalDecomp
+		nTrail := chars[r].nTrailingNonStarters
+		nLead := chars[r].nLeadingNonStarters
+		if tccc > 0 || lccc > 0 || nTrail > 0 {
+			tccc <<= 2
+			tccc |= nTrail
+			s += string([]byte{tccc})
+			index = endMulti
+			for _, r := range d[1:] {
+				if ccc(r) == 0 {
+					index = firstCCC
+				}
+			}
+			if lccc > 0 || nLead > 0 {
+				s += string([]byte{lccc})
+				if index == firstCCC {
+					log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
+				}
+				index = firstLeadingCCC
+			}
+			if cc != lccc {
+				if cc != 0 {
+					log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
+				}
+				index = firstCCCZeroExcept
+			}
+		} else if len(d) > 1 {
+			index = firstMulti
+		}
+		return index, s
+	}
+
+	decompSet := makeDecompSet()
+	const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail.
+	decompSet.insert(firstStarterWithNLead, nLeadStr)
+
+	// Store the uniqued decompositions in a byte buffer,
+	// preceded by their byte length.
+	for _, c := range chars {
+		for _, f := range c.forms {
+			if len(f.expandedDecomp) == 0 {
+				continue
+			}
+			if f.combinesBackward {
+				log.Fatalf("%U: combinesBackward and decompose", c.codePoint)
+			}
+			index, s := mkstr(c.codePoint, &f)
+			decompSet.insert(index, s)
+		}
+	}
+
+	decompositions := bytes.NewBuffer(make([]byte, 0, 10000))
+	size := 0
+	positionMap := make(map[string]uint16)
+	decompositions.WriteString("\000")
+	fmt.Fprintln(w, "const (")
+	for i, m := range decompSet {
+		sa := []string{}
+		for s := range m {
+			sa = append(sa, s)
+		}
+		sort.Strings(sa)
+		for _, s := range sa {
+			p := decompositions.Len()
+			decompositions.WriteString(s)
+			positionMap[s] = uint16(p)
+		}
+		if cname[i] != "" {
+			fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
+		}
+	}
+	fmt.Fprintln(w, "maxDecomp = 0x8000")
+	fmt.Fprintln(w, ")")
+	b := decompositions.Bytes()
+	printBytes(w, b, "decomps")
+	size += len(b)
+
+	varnames := []string{"nfc", "nfkc"}
+	for i := 0; i < FNumberOfFormTypes; i++ {
+		trie := triegen.NewTrie(varnames[i])
+
+		for r, c := range chars {
+			f := c.forms[i]
+			d := f.expandedDecomp
+			if len(d) != 0 {
+				_, key := mkstr(c.codePoint, &f)
+				trie.Insert(rune(r), uint64(positionMap[key]))
+				if c.ccc != ccc(d[0]) {
+					// We assume the lead ccc of a decomposition !=0 in this case.
+					if ccc(d[0]) == 0 {
+						log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
+					}
+				}
+			} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
+				// Handle cases where it can't be detected that the nLead should be equal
+				// to nTrail.
+				trie.Insert(c.codePoint, uint64(positionMap[nLeadStr]))
+			} else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 {
+				trie.Insert(c.codePoint, uint64(0x8000|v))
+			}
+		}
+		sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
+		if err != nil {
+			log.Fatal(err)
+		}
+		size += sz
+	}
+	return size
+}
+
+func contains(sa []string, s string) bool {
+	for _, a := range sa {
+		if a == s {
+			return true
+		}
+	}
+	return false
+}
+
+func makeTables() {
+	w := &bytes.Buffer{}
+
+	size := 0
+	if *tablelist == "" {
+		return
+	}
+	list := strings.Split(*tablelist, ",")
+	if *tablelist == "all" {
+		list = []string{"recomp", "info"}
+	}
+
+	// Compute maximum decomposition size.
+	max := 0
+	for _, c := range chars {
+		if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max {
+			max = n
+		}
+	}
+
+	fmt.Fprintln(w, "const (")
+	fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
+	fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
+	fmt.Fprintln(w)
+	fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
+	fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
+	fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
+	fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
+	fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
+	fmt.Fprintln(w, ")\n")
+
+	// Print the CCC remap table.
+	size += len(cccMap)
+	fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
+	for i := 0; i < len(cccMap); i++ {
+		if i%8 == 0 {
+			fmt.Fprintln(w)
+		}
+		fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
+	}
+	fmt.Fprintln(w, "\n}\n")
+
+	if contains(list, "info") {
+		size += printCharInfoTables(w)
+	}
+
+	if contains(list, "recomp") {
+		// Note that we use 32 bit keys, instead of 64 bit.
+		// This clips the bits of three entries, but we know
+		// this won't cause a collision. The compiler will catch
+		// any changes made to UnicodeData.txt that introduces
+		// a collision.
+		// Note that the recomposition map for NFC and NFKC
+		// are identical.
+
+		// Recomposition map
+		nrentries := 0
+		for _, c := range chars {
+			f := c.forms[FCanonical]
+			if !f.isOneWay && len(f.decomp) > 0 {
+				nrentries++
+			}
+		}
+		sz := nrentries * 8
+		size += sz
+		fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
+		fmt.Fprintln(w, "var recompMap = map[uint32]rune{")
+		for i, c := range chars {
+			f := c.forms[FCanonical]
+			d := f.decomp
+			if !f.isOneWay && len(d) > 0 {
+				key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
+				fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i)
+			}
+		}
+		fmt.Fprintf(w, "}\n\n")
+	}
+
+	fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
+	gen.WriteGoFile("tables.go", "norm", w.Bytes())
+}
+
+func printChars() {
+	if *verbose {
+		for _, c := range chars {
+			if !c.isValid() || c.state == SMissing {
+				continue
+			}
+			fmt.Println(c)
+		}
+	}
+}
+
+// verifyComputed does various consistency tests.
+func verifyComputed() {
+	for i, c := range chars {
+		for _, f := range c.forms {
+			isNo := (f.quickCheck[MDecomposed] == QCNo)
+			if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) {
+				log.Fatalf("%U: NF*D QC must be No if rune decomposes", i)
+			}
+
+			isMaybe := f.quickCheck[MComposed] == QCMaybe
+			if f.combinesBackward != isMaybe {
+				log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i)
+			}
+			if len(f.decomp) > 0 && f.combinesForward && isMaybe {
+				log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i)
+			}
+
+			if len(f.expandedDecomp) != 0 {
+				continue
+			}
+			if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b {
+				// We accept these runes to be treated differently (it only affects
+				// segment breaking in iteration, most likely on improper use), but
+				// reconsider if more characters are added.
+				// U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
+				// U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
+				// U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
+				// U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
+				// U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
+				// U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
+				if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) {
+					log.Fatalf("%U: nLead was %v; want %v", i, a, b)
+				}
+			}
+		}
+		nfc := c.forms[FCanonical]
+		nfkc := c.forms[FCompatibility]
+		if nfc.combinesBackward != nfkc.combinesBackward {
+			log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
+		}
+	}
+}
+
+// Use values in DerivedNormalizationProps.txt to compare against the
+// values we computed.
+// DerivedNormalizationProps.txt has form:
+// 00C0..00C5    ; NFD_QC; N # ...
+// 0374          ; NFD_QC; N # ...
+// See http://unicode.org/reports/tr44/ for full explanation
+func testDerived() {
+	f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(0)
+		c := &chars[r]
+
+		var ftype, mode int
+		qt := p.String(1)
+		switch qt {
+		case "NFC_QC":
+			ftype, mode = FCanonical, MComposed
+		case "NFD_QC":
+			ftype, mode = FCanonical, MDecomposed
+		case "NFKC_QC":
+			ftype, mode = FCompatibility, MComposed
+		case "NFKD_QC":
+			ftype, mode = FCompatibility, MDecomposed
+		default:
+			continue
+		}
+		var qr QCResult
+		switch p.String(2) {
+		case "Y":
+			qr = QCYes
+		case "N":
+			qr = QCNo
+		case "M":
+			qr = QCMaybe
+		default:
+			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
+		}
+		if got := c.forms[ftype].quickCheck[mode]; got != qr {
+			log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
+		}
+		c.forms[ftype].verified[mode] = true
+	}
+	if err := p.Err(); err != nil {
+		log.Fatal(err)
+	}
+	// Any unspecified value must be QCYes. Verify this.
+	for i, c := range chars {
+		for j, fd := range c.forms {
+			for k, qr := range fd.quickCheck {
+				if !fd.verified[k] && qr != QCYes {
+					m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
+					log.Printf(m, i, j, k, qr, c.name)
+				}
+			}
+		}
+	}
+}
+
+var testHeader = `const (
+	Yes = iota
+	No
+	Maybe
+)
+
+type formData struct {
+	qc              uint8
+	combinesForward bool
+	decomposition   string
+}
+
+type runeData struct {
+	r      rune
+	ccc    uint8
+	nLead  uint8
+	nTrail uint8
+	f      [2]formData // 0: canonical; 1: compatibility
+}
+
+func f(qc uint8, cf bool, dec string) [2]formData {
+	return [2]formData{{qc, cf, dec}, {qc, cf, dec}}
+}
+
+func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData {
+	return [2]formData{{qc, cf, d}, {qck, cfk, dk}}
+}
+
+var testData = []runeData{
+`
+
+func printTestdata() {
+	type lastInfo struct {
+		ccc    uint8
+		nLead  uint8
+		nTrail uint8
+		f      string
+	}
+
+	last := lastInfo{}
+	w := &bytes.Buffer{}
+	fmt.Fprintf(w, testHeader)
+	for r, c := range chars {
+		f := c.forms[FCanonical]
+		qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
+		f = c.forms[FCompatibility]
+		qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
+		s := ""
+		if d == dk && qc == qck && cf == cfk {
+			s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d)
+		} else {
+			s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk)
+		}
+		current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
+		if last != current {
+			fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
+			last = current
+		}
+	}
+	fmt.Fprintln(w, "}")
+	gen.WriteGoFile("data_test.go", "norm", w.Bytes())
+}

vendor/golang.org/x/text/unicode/norm/normalize.go

@@ -0,0 +1,609 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Note: the file data_test.go that is generated should not be checked in.
+//go:generate go run maketables.go triegen.go
+//go:generate go test -tags test
+
+// Package norm contains types and functions for normalizing Unicode strings.
+package norm // import "golang.org/x/text/unicode/norm"
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// A Form denotes a canonical representation of Unicode code points.
+// The Unicode-defined normalization and equivalence forms are:
+//
+//   NFC   Unicode Normalization Form C
+//   NFD   Unicode Normalization Form D
+//   NFKC  Unicode Normalization Form KC
+//   NFKD  Unicode Normalization Form KD
+//
+// For a Form f, this documentation uses the notation f(x) to mean
+// the bytes or string x converted to the given form.
+// A position n in x is called a boundary if conversion to the form can
+// proceed independently on both sides:
+//   f(x) == append(f(x[0:n]), f(x[n:])...)
+//
+// References: http://unicode.org/reports/tr15/ and
+// http://unicode.org/notes/tn5/.
+type Form int
+
+const (
+	NFC Form = iota
+	NFD
+	NFKC
+	NFKD
+)
+
+// Bytes returns f(b). May return b if f(b) = b.
+func (f Form) Bytes(b []byte) []byte {
+	src := inputBytes(b)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return b
+	}
+	out := make([]byte, n, len(b))
+	copy(out, b[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b), out: out, flushF: appendFlush}
+	return doAppendInner(&rb, n)
+}
+
+// String returns f(s).
+func (f Form) String(s string) string {
+	src := inputString(s)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return s
+	}
+	out := make([]byte, n, len(s))
+	copy(out, s[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s), out: out, flushF: appendFlush}
+	return string(doAppendInner(&rb, n))
+}
+
+// IsNormal returns true if b == f(b).
+func (f Form) IsNormal(b []byte) bool {
+	src := inputBytes(b)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b)}
+	rb.setFlusher(nil, cmpNormalBytes)
+	for bp < len(b) {
+		rb.out = b[bp:]
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(b), true)
+	}
+	return true
+}
+
+func cmpNormalBytes(rb *reorderBuffer) bool {
+	b := rb.out
+	for i := 0; i < rb.nrune; i++ {
+		info := rb.rune[i]
+		if int(info.size) > len(b) {
+			return false
+		}
+		p := info.pos
+		pe := p + info.size
+		for ; p < pe; p++ {
+			if b[0] != rb.byte[p] {
+				return false
+			}
+			b = b[1:]
+		}
+	}
+	return true
+}
+
+// IsNormalString returns true if s == f(s).
+func (f Form) IsNormalString(s string) bool {
+	src := inputString(s)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s)}
+	rb.setFlusher(nil, func(rb *reorderBuffer) bool {
+		for i := 0; i < rb.nrune; i++ {
+			info := rb.rune[i]
+			if bp+int(info.size) > len(s) {
+				return false
+			}
+			p := info.pos
+			pe := p + info.size
+			for ; p < pe; p++ {
+				if s[bp] != rb.byte[p] {
+					return false
+				}
+				bp++
+			}
+		}
+		return true
+	})
+	for bp < len(s) {
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(s), true)
+	}
+	return true
+}
+
+// patchTail fixes a case where a rune may be incorrectly normalized
+// if it is followed by illegal continuation bytes. It returns the
+// patched buffer and whether the decomposition is still in progress.
+func patchTail(rb *reorderBuffer) bool {
+	info, p := lastRuneStart(&rb.f, rb.out)
+	if p == -1 || info.size == 0 {
+		return true
+	}
+	end := p + int(info.size)
+	extra := len(rb.out) - end
+	if extra > 0 {
+		// Potentially allocating memory. However, this only
+		// happens with ill-formed UTF-8.
+		x := make([]byte, 0)
+		x = append(x, rb.out[len(rb.out)-extra:]...)
+		rb.out = rb.out[:end]
+		decomposeToLastBoundary(rb)
+		rb.doFlush()
+		rb.out = append(rb.out, x...)
+		return false
+	}
+	buf := rb.out[p:]
+	rb.out = rb.out[:p]
+	decomposeToLastBoundary(rb)
+	if s := rb.ss.next(info); s == ssStarter {
+		rb.doFlush()
+		rb.ss.first(info)
+	} else if s == ssOverflow {
+		rb.doFlush()
+		rb.insertCGJ()
+		rb.ss = 0
+	}
+	rb.insertUnsafe(inputBytes(buf), 0, info)
+	return true
+}
+
+func appendQuick(rb *reorderBuffer, i int) int {
+	if rb.nsrc == i {
+		return i
+	}
+	end, _ := rb.f.quickSpan(rb.src, i, rb.nsrc, true)
+	rb.out = rb.src.appendSlice(rb.out, i, end)
+	return end
+}
+
+// Append returns f(append(out, b...)).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) Append(out []byte, src ...byte) []byte {
+	return f.doAppend(out, inputBytes(src), len(src))
+}
+
+func (f Form) doAppend(out []byte, src input, n int) []byte {
+	if n == 0 {
+		return out
+	}
+	ft := formTable[f]
+	// Attempt to do a quickSpan first so we can avoid initializing the reorderBuffer.
+	if len(out) == 0 {
+		p, _ := ft.quickSpan(src, 0, n, true)
+		out = src.appendSlice(out, 0, p)
+		if p == n {
+			return out
+		}
+		rb := reorderBuffer{f: *ft, src: src, nsrc: n, out: out, flushF: appendFlush}
+		return doAppendInner(&rb, p)
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: n}
+	return doAppend(&rb, out, 0)
+}
+
+func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
+	rb.setFlusher(out, appendFlush)
+	src, n := rb.src, rb.nsrc
+	doMerge := len(out) > 0
+	if q := src.skipContinuationBytes(p); q > p {
+		// Move leading non-starters to destination.
+		rb.out = src.appendSlice(rb.out, p, q)
+		p = q
+		doMerge = patchTail(rb)
+	}
+	fd := &rb.f
+	if doMerge {
+		var info Properties
+		if p < n {
+			info = fd.info(src, p)
+			if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 {
+				if p == 0 {
+					decomposeToLastBoundary(rb)
+				}
+				p = decomposeSegment(rb, p, true)
+			}
+		}
+		if info.size == 0 {
+			rb.doFlush()
+			// Append incomplete UTF-8 encoding.
+			return src.appendSlice(rb.out, p, n)
+		}
+		if rb.nrune > 0 {
+			return doAppendInner(rb, p)
+		}
+	}
+	p = appendQuick(rb, p)
+	return doAppendInner(rb, p)
+}
+
+func doAppendInner(rb *reorderBuffer, p int) []byte {
+	for n := rb.nsrc; p < n; {
+		p = decomposeSegment(rb, p, true)
+		p = appendQuick(rb, p)
+	}
+	return rb.out
+}
+
+// AppendString returns f(append(out, []byte(s))).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) AppendString(out []byte, src string) []byte {
+	return f.doAppend(out, inputString(src), len(src))
+}
+
+// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpan(b []byte) int {
+	n, _ := formTable[f].quickSpan(inputBytes(b), 0, len(b), true)
+	return n
+}
+
+// Span implements transform.SpanningTransformer. It returns a boundary n such
+// that b[0:n] == f(b[0:n]). It is not guaranteed to return the largest such n.
+func (f Form) Span(b []byte, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputBytes(b), 0, len(b), atEOF)
+	if n < len(b) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
+// SpanString returns a boundary n such that s[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) SpanString(s string, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputString(s), 0, len(s), atEOF)
+	if n < len(s) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
+// quickSpan returns a boundary n such that src[0:n] == f(src[0:n]) and
+// whether any non-normalized parts were found. If atEOF is false, n will
+// not point past the last segment if this segment might be become
+// non-normalized by appending other runes.
+func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool) {
+	var lastCC uint8
+	ss := streamSafe(0)
+	lastSegStart := i
+	for n = end; i < n; {
+		if j := src.skipASCII(i, n); i != j {
+			i = j
+			lastSegStart = i - 1
+			lastCC = 0
+			ss = 0
+			continue
+		}
+		info := f.info(src, i)
+		if info.size == 0 {
+			if atEOF {
+				// include incomplete runes
+				return n, true
+			}
+			return lastSegStart, true
+		}
+		// This block needs to be before the next, because it is possible to
+		// have an overflow for runes that are starters (e.g. with U+FF9E).
+		switch ss.next(info) {
+		case ssStarter:
+			ss.first(info)
+			lastSegStart = i
+		case ssOverflow:
+			return lastSegStart, false
+		case ssSuccess:
+			if lastCC > info.ccc {
+				return lastSegStart, false
+			}
+		}
+		if f.composing {
+			if !info.isYesC() {
+				break
+			}
+		} else {
+			if !info.isYesD() {
+				break
+			}
+		}
+		lastCC = info.ccc
+		i += int(info.size)
+	}
+	if i == n {
+		if !atEOF {
+			n = lastSegStart
+		}
+		return n, true
+	}
+	return lastSegStart, false
+}
+
+// QuickSpanString returns a boundary n such that s[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpanString(s string) int {
+	n, _ := formTable[f].quickSpan(inputString(s), 0, len(s), true)
+	return n
+}
+
+// FirstBoundary returns the position i of the first boundary in b
+// or -1 if b contains no boundary.
+func (f Form) FirstBoundary(b []byte) int {
+	return f.firstBoundary(inputBytes(b), len(b))
+}
+
+func (f Form) firstBoundary(src input, nsrc int) int {
+	i := src.skipContinuationBytes(0)
+	if i >= nsrc {
+		return -1
+	}
+	fd := formTable[f]
+	ss := streamSafe(0)
+	// We should call ss.first here, but we can't as the first rune is
+	// skipped already. This means FirstBoundary can't really determine
+	// CGJ insertion points correctly. Luckily it doesn't have to.
+	for {
+		info := fd.info(src, i)
+		if info.size == 0 {
+			return -1
+		}
+		if s := ss.next(info); s != ssSuccess {
+			return i
+		}
+		i += int(info.size)
+		if i >= nsrc {
+			if !info.BoundaryAfter() && !ss.isMax() {
+				return -1
+			}
+			return nsrc
+		}
+	}
+}
+
+// FirstBoundaryInString returns the position i of the first boundary in s
+// or -1 if s contains no boundary.
+func (f Form) FirstBoundaryInString(s string) int {
+	return f.firstBoundary(inputString(s), len(s))
+}
+
+// NextBoundary reports the index of the boundary between the first and next
+// segment in b or -1 if atEOF is false and there are not enough bytes to
+// determine this boundary.
+func (f Form) NextBoundary(b []byte, atEOF bool) int {
+	return f.nextBoundary(inputBytes(b), len(b), atEOF)
+}
+
+// NextBoundaryInString reports the index of the boundary between the first and
+// next segment in b or -1 if atEOF is false and there are not enough bytes to
+// determine this boundary.
+func (f Form) NextBoundaryInString(s string, atEOF bool) int {
+	return f.nextBoundary(inputString(s), len(s), atEOF)
+}
+
+func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
+	if nsrc == 0 {
+		if atEOF {
+			return 0
+		}
+		return -1
+	}
+	fd := formTable[f]
+	info := fd.info(src, 0)
+	if info.size == 0 {
+		if atEOF {
+			return 1
+		}
+		return -1
+	}
+	ss := streamSafe(0)
+	ss.first(info)
+
+	for i := int(info.size); i < nsrc; i += int(info.size) {
+		info = fd.info(src, i)
+		if info.size == 0 {
+			if atEOF {
+				return i
+			}
+			return -1
+		}
+		if s := ss.next(info); s != ssSuccess {
+			return i
+		}
+	}
+	if !atEOF && !info.BoundaryAfter() && !ss.isMax() {
+		return -1
+	}
+	return nsrc
+}
+
+// LastBoundary returns the position i of the last boundary in b
+// or -1 if b contains no boundary.
+func (f Form) LastBoundary(b []byte) int {
+	return lastBoundary(formTable[f], b)
+}
+
+func lastBoundary(fd *formInfo, b []byte) int {
+	i := len(b)
+	info, p := lastRuneStart(fd, b)
+	if p == -1 {
+		return -1
+	}
+	if info.size == 0 { // ends with incomplete rune
+		if p == 0 { // starts with incomplete rune
+			return -1
+		}
+		i = p
+		info, p = lastRuneStart(fd, b[:i])
+		if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter
+			return i
+		}
+	}
+	if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8
+		return i
+	}
+	if info.BoundaryAfter() {
+		return i
+	}
+	ss := streamSafe(0)
+	v := ss.backwards(info)
+	for i = p; i >= 0 && v != ssStarter; i = p {
+		info, p = lastRuneStart(fd, b[:i])
+		if v = ss.backwards(info); v == ssOverflow {
+			break
+		}
+		if p+int(info.size) != i {
+			if p == -1 { // no boundary found
+				return -1
+			}
+			return i // boundary after an illegal UTF-8 encoding
+		}
+	}
+	return i
+}
+
+// decomposeSegment scans the first segment in src into rb. It inserts 0x034f
+// (Grapheme Joiner) when it encounters a sequence of more than 30 non-starters
+// and returns the number of bytes consumed from src or iShortDst or iShortSrc.
+func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
+	// Force one character to be consumed.
+	info := rb.f.info(rb.src, sp)
+	if info.size == 0 {
+		return 0
+	}
+	if rb.nrune > 0 {
+		if s := rb.ss.next(info); s == ssStarter {
+			goto end
+		} else if s == ssOverflow {
+			rb.insertCGJ()
+			goto end
+		}
+	} else {
+		rb.ss.first(info)
+	}
+	if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+		return int(err)
+	}
+	for {
+		sp += int(info.size)
+		if sp >= rb.nsrc {
+			if !atEOF && !info.BoundaryAfter() {
+				return int(iShortSrc)
+			}
+			break
+		}
+		info = rb.f.info(rb.src, sp)
+		if info.size == 0 {
+			if !atEOF {
+				return int(iShortSrc)
+			}
+			break
+		}
+		if s := rb.ss.next(info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			rb.insertCGJ()
+			break
+		}
+		if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+			return int(err)
+		}
+	}
+end:
+	if !rb.doFlush() {
+		return int(iShortDst)
+	}
+	return sp
+}
+
+// lastRuneStart returns the runeInfo and position of the last
+// rune in buf or the zero runeInfo and -1 if no rune was found.
+func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) {
+	p := len(buf) - 1
+	for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- {
+	}
+	if p < 0 {
+		return Properties{}, -1
+	}
+	return fd.info(inputBytes(buf), p), p
+}
+
+// decomposeToLastBoundary finds an open segment at the end of the buffer
+// and scans it into rb. Returns the buffer minus the last segment.
+func decomposeToLastBoundary(rb *reorderBuffer) {
+	fd := &rb.f
+	info, i := lastRuneStart(fd, rb.out)
+	if int(info.size) != len(rb.out)-i {
+		// illegal trailing continuation bytes
+		return
+	}
+	if info.BoundaryAfter() {
+		return
+	}
+	var add [maxNonStarters + 1]Properties // stores runeInfo in reverse order
+	padd := 0
+	ss := streamSafe(0)
+	p := len(rb.out)
+	for {
+		add[padd] = info
+		v := ss.backwards(info)
+		if v == ssOverflow {
+			// Note that if we have an overflow, it the string we are appending to
+			// is not correctly normalized. In this case the behavior is undefined.
+			break
+		}
+		padd++
+		p -= int(info.size)
+		if v == ssStarter || p < 0 {
+			break
+		}
+		info, i = lastRuneStart(fd, rb.out[:p])
+		if int(info.size) != p-i {
+			break
+		}
+	}
+	rb.ss = ss
+	// Copy bytes for insertion as we may need to overwrite rb.out.
+	var buf [maxBufferSize * utf8.UTFMax]byte
+	cp := buf[:copy(buf[:], rb.out[p:])]
+	rb.out = rb.out[:p]
+	for padd--; padd >= 0; padd-- {
+		info = add[padd]
+		rb.insertUnsafe(inputBytes(cp), 0, info)
+		cp = cp[info.size:]
+	}
+}

vendor/golang.org/x/text/unicode/norm/readwriter.go

@@ -0,0 +1,125 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "io"
+
+type normWriter struct {
+	rb  reorderBuffer
+	w   io.Writer
+	buf []byte
+}
+
+// Write implements the standard write interface.  If the last characters are
+// not at a normalization boundary, the bytes will be buffered for the next
+// write. The remaining bytes will be written on close.
+func (w *normWriter) Write(data []byte) (n int, err error) {
+	// Process data in pieces to keep w.buf size bounded.
+	const chunk = 4000
+
+	for len(data) > 0 {
+		// Normalize into w.buf.
+		m := len(data)
+		if m > chunk {
+			m = chunk
+		}
+		w.rb.src = inputBytes(data[:m])
+		w.rb.nsrc = m
+		w.buf = doAppend(&w.rb, w.buf, 0)
+		data = data[m:]
+		n += m
+
+		// Write out complete prefix, save remainder.
+		// Note that lastBoundary looks back at most 31 runes.
+		i := lastBoundary(&w.rb.f, w.buf)
+		if i == -1 {
+			i = 0
+		}
+		if i > 0 {
+			if _, err = w.w.Write(w.buf[:i]); err != nil {
+				break
+			}
+			bn := copy(w.buf, w.buf[i:])
+			w.buf = w.buf[:bn]
+		}
+	}
+	return n, err
+}
+
+// Close forces data that remains in the buffer to be written.
+func (w *normWriter) Close() error {
+	if len(w.buf) > 0 {
+		_, err := w.w.Write(w.buf)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Writer returns a new writer that implements Write(b)
+// by writing f(b) to w.  The returned writer may use an
+// an internal buffer to maintain state across Write calls.
+// Calling its Close method writes any buffered data to w.
+func (f Form) Writer(w io.Writer) io.WriteCloser {
+	wr := &normWriter{rb: reorderBuffer{}, w: w}
+	wr.rb.init(f, nil)
+	return wr
+}
+
+type normReader struct {
+	rb           reorderBuffer
+	r            io.Reader
+	inbuf        []byte
+	outbuf       []byte
+	bufStart     int
+	lastBoundary int
+	err          error
+}
+
+// Read implements the standard read interface.
+func (r *normReader) Read(p []byte) (int, error) {
+	for {
+		if r.lastBoundary-r.bufStart > 0 {
+			n := copy(p, r.outbuf[r.bufStart:r.lastBoundary])
+			r.bufStart += n
+			if r.lastBoundary-r.bufStart > 0 {
+				return n, nil
+			}
+			return n, r.err
+		}
+		if r.err != nil {
+			return 0, r.err
+		}
+		outn := copy(r.outbuf, r.outbuf[r.lastBoundary:])
+		r.outbuf = r.outbuf[0:outn]
+		r.bufStart = 0
+
+		n, err := r.r.Read(r.inbuf)
+		r.rb.src = inputBytes(r.inbuf[0:n])
+		r.rb.nsrc, r.err = n, err
+		if n > 0 {
+			r.outbuf = doAppend(&r.rb, r.outbuf, 0)
+		}
+		if err == io.EOF {
+			r.lastBoundary = len(r.outbuf)
+		} else {
+			r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf)
+			if r.lastBoundary == -1 {
+				r.lastBoundary = 0
+			}
+		}
+	}
+}
+
+// Reader returns a new reader that implements Read
+// by reading data from r and returning f(data).
+func (f Form) Reader(r io.Reader) io.Reader {
+	const chunk = 4000
+	buf := make([]byte, chunk)
+	rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf}
+	rr.rb.init(f, buf)
+	return rr
+}

vendor/golang.org/x/text/unicode/norm/transform.go

@@ -0,0 +1,88 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// Reset implements the Reset method of the transform.Transformer interface.
+func (Form) Reset() {}
+
+// Transform implements the Transform method of the transform.Transformer
+// interface. It may need to write segments of up to MaxSegmentSize at once.
+// Users should either catch ErrShortDst and allow dst to grow or have dst be at
+// least of size MaxTransformChunkSize to be guaranteed of progress.
+func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := 0
+	// Cap the maximum number of src bytes to check.
+	b := src
+	eof := atEOF
+	if ns := len(dst); ns < len(b) {
+		err = transform.ErrShortDst
+		eof = false
+		b = b[:ns]
+	}
+	i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof)
+	n += copy(dst[n:], b[n:i])
+	if !ok {
+		nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF)
+		return nDst + n, nSrc + n, err
+	}
+	if n < len(src) && !atEOF {
+		err = transform.ErrShortSrc
+	}
+	return n, n, err
+}
+
+func flushTransform(rb *reorderBuffer) bool {
+	// Write out (must fully fit in dst, or else it is a ErrShortDst).
+	if len(rb.out) < rb.nrune*utf8.UTFMax {
+		return false
+	}
+	rb.out = rb.out[rb.flushCopy(rb.out):]
+	return true
+}
+
+var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc}
+
+// transform implements the transform.Transformer interface. It is only called
+// when quickSpan does not pass for a given string.
+func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// TODO: get rid of reorderBuffer. See CL 23460044.
+	rb := reorderBuffer{}
+	rb.init(f, src)
+	for {
+		// Load segment into reorder buffer.
+		rb.setFlusher(dst[nDst:], flushTransform)
+		end := decomposeSegment(&rb, nSrc, atEOF)
+		if end < 0 {
+			return nDst, nSrc, errs[-end]
+		}
+		nDst = len(dst) - len(rb.out)
+		nSrc = end
+
+		// Next quickSpan.
+		end = rb.nsrc
+		eof := atEOF
+		if n := nSrc + len(dst) - nDst; n < end {
+			err = transform.ErrShortDst
+			end = n
+			eof = false
+		}
+		end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof)
+		n := copy(dst[nDst:], rb.src.bytes[nSrc:end])
+		nSrc += n
+		nDst += n
+		if ok {
+			if n < rb.nsrc && !atEOF {
+				err = transform.ErrShortSrc
+			}
+			return nDst, nSrc, err
+		}
+	}
+}

vendor/golang.org/x/text/unicode/norm/trie.go

@@ -0,0 +1,54 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type sparseBlocks struct {
+	values []valueRange
+	offset []uint16
+}
+
+var nfcSparse = sparseBlocks{
+	values: nfcSparseValues[:],
+	offset: nfcSparseOffset[:],
+}
+
+var nfkcSparse = sparseBlocks{
+	values: nfkcSparseValues[:],
+	offset: nfkcSparseOffset[:],
+}
+
+var (
+	nfcData  = newNfcTrie(0)
+	nfkcData = newNfkcTrie(0)
+)
+
+// lookupValue determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	offset := t.offset[n]
+	header := t.values[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}

vendor/golang.org/x/text/unicode/norm/triegen.go

@@ -0,0 +1,117 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Trie table generator.
+// Used by make*tables tools to generate a go file with trie data structures
+// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
+// sequence are used to lookup offsets in the index table to be used for the
+// next byte. The last byte is used to index into a table with 16-bit values.
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+const maxSparseEntries = 16
+
+type normCompacter struct {
+	sparseBlocks [][]uint64
+	sparseOffset []uint16
+	sparseCount  int
+	name         string
+}
+
+func mostFrequentStride(a []uint64) int {
+	counts := make(map[int]int)
+	var v int
+	for _, x := range a {
+		if stride := int(x) - v; v != 0 && stride >= 0 {
+			counts[stride]++
+		}
+		v = int(x)
+	}
+	var maxs, maxc int
+	for stride, cnt := range counts {
+		if cnt > maxc || (cnt == maxc && stride < maxs) {
+			maxs, maxc = stride, cnt
+		}
+	}
+	return maxs
+}
+
+func countSparseEntries(a []uint64) int {
+	stride := mostFrequentStride(a)
+	var v, count int
+	for _, tv := range a {
+		if int(tv)-v != stride {
+			if tv != 0 {
+				count++
+			}
+		}
+		v = int(tv)
+	}
+	return count
+}
+
+func (c *normCompacter) Size(v []uint64) (sz int, ok bool) {
+	if n := countSparseEntries(v); n <= maxSparseEntries {
+		return (n+1)*4 + 2, true
+	}
+	return 0, false
+}
+
+func (c *normCompacter) Store(v []uint64) uint32 {
+	h := uint32(len(c.sparseOffset))
+	c.sparseBlocks = append(c.sparseBlocks, v)
+	c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount))
+	c.sparseCount += countSparseEntries(v) + 1
+	return h
+}
+
+func (c *normCompacter) Handler() string {
+	return c.name + "Sparse.lookup"
+}
+
+func (c *normCompacter) Print(w io.Writer) (retErr error) {
+	p := func(f string, x ...interface{}) {
+		if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil {
+			retErr = err
+		}
+	}
+
+	ls := len(c.sparseBlocks)
+	p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2)
+	p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset)
+
+	ns := c.sparseCount
+	p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4)
+	p("var %sSparseValues = [%d]valueRange {", c.name, ns)
+	for i, b := range c.sparseBlocks {
+		p("\n// Block %#x, offset %#x", i, c.sparseOffset[i])
+		var v int
+		stride := mostFrequentStride(b)
+		n := countSparseEntries(b)
+		p("\n{value:%#04x,lo:%#02x},", stride, uint8(n))
+		for i, nv := range b {
+			if int(nv)-v != stride {
+				if v != 0 {
+					p(",hi:%#02x},", 0x80+i-1)
+				}
+				if nv != 0 {
+					p("\n{value:%#04x,lo:%#02x", nv, 0x80+i)
+				}
+			}
+			v = int(nv)
+		}
+		if v != 0 {
+			p(",hi:%#02x},", 0x80+len(b)-1)
+		}
+	}
+	p("\n}\n\n")
+	return
+}

vendor/golang.org/x/text/unicode/rangetable/gen.go

@@ -0,0 +1,113 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"reflect"
+	"sort"
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/ucd"
+	"golang.org/x/text/unicode/rangetable"
+)
+
+var versionList = flag.String("versions", "",
+	"list of versions for which to generate RangeTables")
+
+const bootstrapMessage = `No versions specified.
+To bootstrap the code generation, run:
+	go run gen.go --versions=4.1.0,5.0.0,6.0.0,6.1.0,6.2.0,6.3.0,7.0.0
+
+and ensure that the latest versions are included by checking:
+	http://www.unicode.org/Public/`
+
+func getVersions() []string {
+	if *versionList == "" {
+		log.Fatal(bootstrapMessage)
+	}
+
+	versions := strings.Split(*versionList, ",")
+	sort.Strings(versions)
+
+	// Ensure that at least the current version is included.
+	for _, v := range versions {
+		if v == gen.UnicodeVersion() {
+			return versions
+		}
+	}
+
+	versions = append(versions, gen.UnicodeVersion())
+	sort.Strings(versions)
+	return versions
+}
+
+func main() {
+	gen.Init()
+
+	versions := getVersions()
+
+	w := &bytes.Buffer{}
+
+	fmt.Fprintf(w, "//go:generate go run gen.go --versions=%s\n\n", strings.Join(versions, ","))
+	fmt.Fprintf(w, "import \"unicode\"\n\n")
+
+	vstr := func(s string) string { return strings.Replace(s, ".", "_", -1) }
+
+	fmt.Fprintf(w, "var assigned = map[string]*unicode.RangeTable{\n")
+	for _, v := range versions {
+		fmt.Fprintf(w, "\t%q: assigned%s,\n", v, vstr(v))
+	}
+	fmt.Fprintf(w, "}\n\n")
+
+	var size int
+	for _, v := range versions {
+		assigned := []rune{}
+
+		r := gen.Open("http://www.unicode.org/Public/", "", v+"/ucd/UnicodeData.txt")
+		ucd.Parse(r, func(p *ucd.Parser) {
+			assigned = append(assigned, p.Rune(0))
+		})
+
+		rt := rangetable.New(assigned...)
+		sz := int(reflect.TypeOf(unicode.RangeTable{}).Size())
+		sz += int(reflect.TypeOf(unicode.Range16{}).Size()) * len(rt.R16)
+		sz += int(reflect.TypeOf(unicode.Range32{}).Size()) * len(rt.R32)
+
+		fmt.Fprintf(w, "// size %d bytes (%d KiB)\n", sz, sz/1024)
+		fmt.Fprintf(w, "var assigned%s = ", vstr(v))
+		print(w, rt)
+
+		size += sz
+	}
+
+	fmt.Fprintf(w, "// Total size %d bytes (%d KiB)\n", size, size/1024)
+
+	gen.WriteGoFile("tables.go", "rangetable", w.Bytes())
+}
+
+func print(w io.Writer, rt *unicode.RangeTable) {
+	fmt.Fprintln(w, "&unicode.RangeTable{")
+	fmt.Fprintln(w, "\tR16: []unicode.Range16{")
+	for _, r := range rt.R16 {
+		fmt.Fprintf(w, "\t\t{%#04x, %#04x, %d},\n", r.Lo, r.Hi, r.Stride)
+	}
+	fmt.Fprintln(w, "\t},")
+	fmt.Fprintln(w, "\tR32: []unicode.Range32{")
+	for _, r := range rt.R32 {
+		fmt.Fprintf(w, "\t\t{%#08x, %#08x, %d},\n", r.Lo, r.Hi, r.Stride)
+	}
+	fmt.Fprintln(w, "\t},")
+	fmt.Fprintf(w, "\tLatinOffset: %d,\n", rt.LatinOffset)
+	fmt.Fprintf(w, "}\n\n")
+}

vendor/golang.org/x/text/unicode/rangetable/merge.go

@@ -0,0 +1,260 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package rangetable
+
+import (
+	"unicode"
+)
+
+// atEnd is used to mark a completed iteration.
+const atEnd = unicode.MaxRune + 1
+
+// Merge returns a new RangeTable that is the union of the given tables.
+// It can also be used to compact user-created RangeTables. The entries in
+// R16 and R32 for any given RangeTable should be sorted and non-overlapping.
+//
+// A lookup in the resulting table can be several times faster than using In
+// directly on the ranges. Merge is an expensive operation, however, and only
+// makes sense if one intends to use the result for more than a couple of
+// hundred lookups.
+func Merge(ranges ...*unicode.RangeTable) *unicode.RangeTable {
+	rt := &unicode.RangeTable{}
+	if len(ranges) == 0 {
+		return rt
+	}
+
+	iter := tablesIter(make([]tableIndex, len(ranges)))
+
+	for i, t := range ranges {
+		iter[i] = tableIndex{t, 0, atEnd}
+		if len(t.R16) > 0 {
+			iter[i].next = rune(t.R16[0].Lo)
+		}
+	}
+
+	if r0 := iter.next16(); r0.Stride != 0 {
+		for {
+			r1 := iter.next16()
+			if r1.Stride == 0 {
+				rt.R16 = append(rt.R16, r0)
+				break
+			}
+			stride := r1.Lo - r0.Hi
+			if (r1.Lo == r1.Hi || stride == r1.Stride) && (r0.Lo == r0.Hi || stride == r0.Stride) {
+				// Fully merge the next range into the previous one.
+				r0.Hi, r0.Stride = r1.Hi, stride
+				continue
+			} else if stride == r0.Stride {
+				// Move the first element of r1 to r0. This may eliminate an
+				// entry.
+				r0.Hi = r1.Lo
+				r0.Stride = stride
+				r1.Lo = r1.Lo + r1.Stride
+				if r1.Lo > r1.Hi {
+					continue
+				}
+			}
+			rt.R16 = append(rt.R16, r0)
+			r0 = r1
+		}
+	}
+
+	for i, t := range ranges {
+		iter[i] = tableIndex{t, 0, atEnd}
+		if len(t.R32) > 0 {
+			iter[i].next = rune(t.R32[0].Lo)
+		}
+	}
+
+	if r0 := iter.next32(); r0.Stride != 0 {
+		for {
+			r1 := iter.next32()
+			if r1.Stride == 0 {
+				rt.R32 = append(rt.R32, r0)
+				break
+			}
+			stride := r1.Lo - r0.Hi
+			if (r1.Lo == r1.Hi || stride == r1.Stride) && (r0.Lo == r0.Hi || stride == r0.Stride) {
+				// Fully merge the next range into the previous one.
+				r0.Hi, r0.Stride = r1.Hi, stride
+				continue
+			} else if stride == r0.Stride {
+				// Move the first element of r1 to r0. This may eliminate an
+				// entry.
+				r0.Hi = r1.Lo
+				r1.Lo = r1.Lo + r1.Stride
+				if r1.Lo > r1.Hi {
+					continue
+				}
+			}
+			rt.R32 = append(rt.R32, r0)
+			r0 = r1
+		}
+	}
+
+	for i := 0; i < len(rt.R16) && rt.R16[i].Hi <= unicode.MaxLatin1; i++ {
+		rt.LatinOffset = i + 1
+	}
+
+	return rt
+}
+
+type tableIndex struct {
+	t    *unicode.RangeTable
+	p    uint32
+	next rune
+}
+
+type tablesIter []tableIndex
+
+// sortIter does an insertion sort using the next field of tableIndex. Insertion
+// sort is a good sorting algorithm for this case.
+func sortIter(t []tableIndex) {
+	for i := range t {
+		for j := i; j > 0 && t[j-1].next > t[j].next; j-- {
+			t[j], t[j-1] = t[j-1], t[j]
+		}
+	}
+}
+
+// next16 finds the ranged to be added to the table. If ranges overlap between
+// multiple tables it clips the result to a non-overlapping range if the
+// elements are not fully subsumed. It returns a zero range if there are no more
+// ranges.
+func (ti tablesIter) next16() unicode.Range16 {
+	sortIter(ti)
+
+	t0 := ti[0]
+	if t0.next == atEnd {
+		return unicode.Range16{}
+	}
+	r0 := t0.t.R16[t0.p]
+	r0.Lo = uint16(t0.next)
+
+	// We restrict the Hi of the current range if it overlaps with another range.
+	for i := range ti {
+		tn := ti[i]
+		// Since our tableIndices are sorted by next, we can break if the there
+		// is no overlap. The first value of a next range can always be merged
+		// into the current one, so we can break in case of equality as well.
+		if rune(r0.Hi) <= tn.next {
+			break
+		}
+		rn := tn.t.R16[tn.p]
+		rn.Lo = uint16(tn.next)
+
+		// Limit r0.Hi based on next ranges in list, but allow it to overlap
+		// with ranges as long as it subsumes it.
+		m := (rn.Lo - r0.Lo) % r0.Stride
+		if m == 0 && (rn.Stride == r0.Stride || rn.Lo == rn.Hi) {
+			// Overlap, take the min of the two Hi values: for simplicity's sake
+			// we only process one range at a time.
+			if r0.Hi > rn.Hi {
+				r0.Hi = rn.Hi
+			}
+		} else {
+			// Not a compatible stride. Set to the last possible value before
+			// rn.Lo, but ensure there is at least one value.
+			if x := rn.Lo - m; r0.Lo <= x {
+				r0.Hi = x
+			}
+			break
+		}
+	}
+
+	// Update the next values for each table.
+	for i := range ti {
+		tn := &ti[i]
+		if rune(r0.Hi) < tn.next {
+			break
+		}
+		rn := tn.t.R16[tn.p]
+		stride := rune(rn.Stride)
+		tn.next += stride * (1 + ((rune(r0.Hi) - tn.next) / stride))
+		if rune(rn.Hi) < tn.next {
+			if tn.p++; int(tn.p) == len(tn.t.R16) {
+				tn.next = atEnd
+			} else {
+				tn.next = rune(tn.t.R16[tn.p].Lo)
+			}
+		}
+	}
+
+	if r0.Lo == r0.Hi {
+		r0.Stride = 1
+	}
+
+	return r0
+}
+
+// next32 finds the ranged to be added to the table. If ranges overlap between
+// multiple tables it clips the result to a non-overlapping range if the
+// elements are not fully subsumed. It returns a zero range if there are no more
+// ranges.
+func (ti tablesIter) next32() unicode.Range32 {
+	sortIter(ti)
+
+	t0 := ti[0]
+	if t0.next == atEnd {
+		return unicode.Range32{}
+	}
+	r0 := t0.t.R32[t0.p]
+	r0.Lo = uint32(t0.next)
+
+	// We restrict the Hi of the current range if it overlaps with another range.
+	for i := range ti {
+		tn := ti[i]
+		// Since our tableIndices are sorted by next, we can break if the there
+		// is no overlap. The first value of a next range can always be merged
+		// into the current one, so we can break in case of equality as well.
+		if rune(r0.Hi) <= tn.next {
+			break
+		}
+		rn := tn.t.R32[tn.p]
+		rn.Lo = uint32(tn.next)
+
+		// Limit r0.Hi based on next ranges in list, but allow it to overlap
+		// with ranges as long as it subsumes it.
+		m := (rn.Lo - r0.Lo) % r0.Stride
+		if m == 0 && (rn.Stride == r0.Stride || rn.Lo == rn.Hi) {
+			// Overlap, take the min of the two Hi values: for simplicity's sake
+			// we only process one range at a time.
+			if r0.Hi > rn.Hi {
+				r0.Hi = rn.Hi
+			}
+		} else {
+			// Not a compatible stride. Set to the last possible value before
+			// rn.Lo, but ensure there is at least one value.
+			if x := rn.Lo - m; r0.Lo <= x {
+				r0.Hi = x
+			}
+			break
+		}
+	}
+
+	// Update the next values for each table.
+	for i := range ti {
+		tn := &ti[i]
+		if rune(r0.Hi) < tn.next {
+			break
+		}
+		rn := tn.t.R32[tn.p]
+		stride := rune(rn.Stride)
+		tn.next += stride * (1 + ((rune(r0.Hi) - tn.next) / stride))
+		if rune(rn.Hi) < tn.next {
+			if tn.p++; int(tn.p) == len(tn.t.R32) {
+				tn.next = atEnd
+			} else {
+				tn.next = rune(tn.t.R32[tn.p].Lo)
+			}
+		}
+	}
+
+	if r0.Lo == r0.Hi {
+		r0.Stride = 1
+	}
+
+	return r0
+}

vendor/golang.org/x/text/unicode/rangetable/rangetable.go

@@ -0,0 +1,70 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package rangetable provides utilities for creating and inspecting
+// unicode.RangeTables.
+package rangetable
+
+import (
+	"sort"
+	"unicode"
+)
+
+// New creates a RangeTable from the given runes, which may contain duplicates.
+func New(r ...rune) *unicode.RangeTable {
+	if len(r) == 0 {
+		return &unicode.RangeTable{}
+	}
+
+	sort.Sort(byRune(r))
+
+	// Remove duplicates.
+	k := 1
+	for i := 1; i < len(r); i++ {
+		if r[k-1] != r[i] {
+			r[k] = r[i]
+			k++
+		}
+	}
+
+	var rt unicode.RangeTable
+	for _, r := range r[:k] {
+		if r <= 0xFFFF {
+			rt.R16 = append(rt.R16, unicode.Range16{Lo: uint16(r), Hi: uint16(r), Stride: 1})
+		} else {
+			rt.R32 = append(rt.R32, unicode.Range32{Lo: uint32(r), Hi: uint32(r), Stride: 1})
+		}
+	}
+
+	// Optimize RangeTable.
+	return Merge(&rt)
+}
+
+type byRune []rune
+
+func (r byRune) Len() int           { return len(r) }
+func (r byRune) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
+func (r byRune) Less(i, j int) bool { return r[i] < r[j] }
+
+// Visit visits all runes in the given RangeTable in order, calling fn for each.
+func Visit(rt *unicode.RangeTable, fn func(rune)) {
+	for _, r16 := range rt.R16 {
+		for r := rune(r16.Lo); r <= rune(r16.Hi); r += rune(r16.Stride) {
+			fn(r)
+		}
+	}
+	for _, r32 := range rt.R32 {
+		for r := rune(r32.Lo); r <= rune(r32.Hi); r += rune(r32.Stride) {
+			fn(r)
+		}
+	}
+}
+
+// Assigned returns a RangeTable with all assigned code points for a given
+// Unicode version. This includes graphic, format, control, and private-use
+// characters. It returns nil if the data for the given version is not
+// available.
+func Assigned(version string) *unicode.RangeTable {
+	return assigned[version]
+}