diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 3664255cd7f..ba29295dc6e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -48,30 +48,130 @@ "ImportPath": "github.com/abbot/go-http-auth", "Rev": "c0ef4539dfab4d21c8ef20ba2924f9fc6f186d35" }, + { + "ImportPath": "github.com/appc/spec/aci", + "Comment": "v0.5.1-55-g87808a3", + "Rev": "87808a37061a4a2e6204ccea5fd2fc930576db94" + }, + { + "ImportPath": "github.com/appc/spec/pkg/acirenderer", + "Comment": "v0.5.1-55-g87808a3", + "Rev": "87808a37061a4a2e6204ccea5fd2fc930576db94" + }, + { + "ImportPath": "github.com/appc/spec/pkg/tarheader", + "Comment": "v0.5.1-55-g87808a3", + "Rev": "87808a37061a4a2e6204ccea5fd2fc930576db94" + }, + { + "ImportPath": "github.com/appc/spec/schema", + "Comment": "v0.5.1-55-g87808a3", + "Rev": "87808a37061a4a2e6204ccea5fd2fc930576db94" + }, { "ImportPath": "github.com/beorn7/perks/quantile", "Rev": "b965b613227fddccbfffe13eae360ed3fa822f8d" }, + { + "ImportPath": "github.com/camlistore/lock", + "Rev": "ae27720f340952636b826119b58130b9c1a847a0" + }, { "ImportPath": "github.com/codegangsta/negroni", "Comment": "v0.1-62-g8d75e11", "Rev": "8d75e11374a1928608c906fe745b538483e7aeb2" }, + { + "ImportPath": "github.com/coreos/etcd/etcdserver/etcdhttp/httptypes", + "Comment": "v2.0.4-288-g866a9d4", + "Rev": "866a9d4e41401657ea44bf539b2c5561d6fdcd67" + }, + { + "ImportPath": "github.com/coreos/etcd/pkg/types", + "Comment": "v2.0.4-288-g866a9d4", + "Rev": "866a9d4e41401657ea44bf539b2c5561d6fdcd67" + }, { "ImportPath": "github.com/coreos/go-etcd/etcd", "Comment": "v2.0.0-3-g0424b5f", "Rev": "0424b5f86ef0ca57a5309c599f74bbb3e97ecd9d" }, + { + "ImportPath": "github.com/coreos/go-semver/semver", + "Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa" + }, { "ImportPath": "github.com/coreos/go-systemd/dbus", "Comment": "v2-27-g97e243d", "Rev": "97e243d21a8e232e9d8af38ba2366dfcfceebeba" }, + { + "ImportPath": "github.com/coreos/go-systemd/unit", + "Comment": "v2-27-g97e243d", + "Rev": "97e243d21a8e232e9d8af38ba2366dfcfceebeba" + }, + { + "ImportPath": "github.com/coreos/rkt/pkg/aci", + "Comment": "v0.5.4", + "Rev": "c8a7050a883653266137ae05f6e8f166db52eb67" + }, + { + "ImportPath": "github.com/coreos/rkt/pkg/lock", + "Comment": "v0.5.4", + "Rev": "c8a7050a883653266137ae05f6e8f166db52eb67" + }, + { + "ImportPath": "github.com/coreos/rkt/pkg/sys", + "Comment": "v0.5.4", + "Rev": "c8a7050a883653266137ae05f6e8f166db52eb67" + }, + { + "ImportPath": "github.com/coreos/rkt/pkg/tar", + "Comment": "v0.5.4", + "Rev": "c8a7050a883653266137ae05f6e8f166db52eb67" + }, + { + "ImportPath": "github.com/coreos/rkt/store", + "Comment": "v0.5.4", + "Rev": "c8a7050a883653266137ae05f6e8f166db52eb67" + }, { "ImportPath": "github.com/cpuguy83/go-md2man/mangen", "Comment": "v1.0.2-5-g2831f11", "Rev": "2831f11f66ff4008f10e2cd7ed9a85e3d3fc2bed" }, + { + "ImportPath": "github.com/cznic/bufs", + "Rev": "3dcccbd7064a1689f9c093a988ea11ac00e21f51" + }, + { + "ImportPath": "github.com/cznic/exp/lldb", + "Rev": "9b0e4be12fbdb7b843e0a658a04c35d160371789" + }, + { + "ImportPath": "github.com/cznic/fileutil", + "Rev": "21ae57c9dce724a15e88bd9cd46d5668f3e880a5" + }, + { + "ImportPath": "github.com/cznic/mathutil", + "Rev": "250d0b9d3304c5ea0c4cfc7d9efc7ee528b81f3b" + }, + { + "ImportPath": "github.com/cznic/ql", + "Rev": "fc1b91b82089d3f132fbed8a7c9f349c3133eb96" + }, + { + "ImportPath": "github.com/cznic/sortutil", + "Rev": "d4401851b4c370f979b842fa1e45e0b3b718b391" + }, + { + "ImportPath": "github.com/cznic/strutil", + "Rev": "97bc31f80ac4c9fa9c5dc5fea74c383858988ea2" + }, + { + "ImportPath": "github.com/cznic/zappy", + "Rev": "47331054e4f96186e3ff772877c0443909368a45" + }, { "ImportPath": "github.com/davecgh/go-spew/spew", "Rev": "3e6e67c4dcea3ac2f25fd4731abc0e1deaf36216" @@ -331,6 +431,14 @@ "Comment": "v1.0-28-g8adf9e1730c5", "Rev": "8adf9e1730c55cdc590de7d49766cb2acc88d8f2" }, + { + "ImportPath": "github.com/petar/GoLLRB/llrb", + "Rev": "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" + }, + { + "ImportPath": "github.com/peterbourgon/diskv", + "Rev": "508f5671a72eeaef05cf8c24abe7fbc1c07faf69" + }, { "ImportPath": "github.com/prometheus/client_golang/model", "Comment": "0.4.0-1-g692492e", @@ -411,6 +519,14 @@ "Comment": "v1.0-13-g5292687", "Rev": "5292687f5379e01054407da44d7c4590a61fd3de" }, + { + "ImportPath": "golang.org/x/crypto/cast5", + "Rev": "a7ead6ddf06233883deca151dffaef2effbf498f" + }, + { + "ImportPath": "golang.org/x/crypto/openpgp", + "Rev": "a7ead6ddf06233883deca151dffaef2effbf498f" + }, { "ImportPath": "golang.org/x/net/context", "Rev": "cbcac7bb8415db9b6cb4d1ebab1dc9afbd688b97" diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/build.go b/Godeps/_workspace/src/github.com/appc/spec/aci/build.go new file mode 100644 index 00000000000..12a24943246 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/build.go @@ -0,0 +1,81 @@ +package aci + +import ( + "archive/tar" + "io" + "os" + "path/filepath" + + "github.com/appc/spec/pkg/tarheader" +) + +// BuildWalker creates a filepath.WalkFunc that walks over the given root +// (which should represent an ACI layout on disk) and adds the files in the +// rootfs/ subdirectory to the given ArchiveWriter +func BuildWalker(root string, aw ArchiveWriter) filepath.WalkFunc { + // cache of inode -> filepath, used to leverage hard links in the archive + inos := map[uint64]string{} + return func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relpath, err := filepath.Rel(root, path) + if err != nil { + return err + } + if relpath == "." { + return nil + } + if relpath == ManifestFile { + // ignore; this will be written by the archive writer + // TODO(jonboulle): does this make sense? maybe just remove from archivewriter? + return nil + } + + link := "" + var r io.Reader + switch info.Mode() & os.ModeType { + case os.ModeSocket: + return nil + case os.ModeNamedPipe: + case os.ModeCharDevice: + case os.ModeDevice: + case os.ModeDir: + case os.ModeSymlink: + target, err := os.Readlink(path) + if err != nil { + return err + } + link = target + default: + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + r = file + } + + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + panic(err) + } + // Because os.FileInfo's Name method returns only the base + // name of the file it describes, it may be necessary to + // modify the Name field of the returned header to provide the + // full path name of the file. + hdr.Name = relpath + tarheader.Populate(hdr, info, inos) + // If the file is a hard link to a file we've already seen, we + // don't need the contents + if hdr.Typeflag == tar.TypeLink { + hdr.Size = 0 + r = nil + } + if err := aw.AddFile(hdr, r); err != nil { + return err + } + + return nil + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/doc.go b/Godeps/_workspace/src/github.com/appc/spec/aci/doc.go new file mode 100644 index 00000000000..b8e49dcc030 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/doc.go @@ -0,0 +1,2 @@ +// Package aci contains various functions for working with App Container Images. +package aci diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/file.go b/Godeps/_workspace/src/github.com/appc/spec/aci/file.go new file mode 100644 index 00000000000..951db81d6c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/file.go @@ -0,0 +1,194 @@ +package aci + +import ( + "archive/tar" + "bytes" + "compress/bzip2" + "compress/gzip" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os/exec" + "path/filepath" + + "github.com/appc/spec/schema" +) + +type FileType string + +const ( + TypeGzip = FileType("gz") + TypeBzip2 = FileType("bz2") + TypeXz = FileType("xz") + TypeTar = FileType("tar") + TypeText = FileType("text") + TypeUnknown = FileType("unknown") + + readLen = 512 // max bytes to sniff + + hexHdrGzip = "1f8b" + hexHdrBzip2 = "425a68" + hexHdrXz = "fd377a585a00" + hexSigTar = "7573746172" + + tarOffset = 257 + + textMime = "text/plain; charset=utf-8" +) + +var ( + hdrGzip []byte + hdrBzip2 []byte + hdrXz []byte + sigTar []byte + tarEnd int +) + +func mustDecodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +func init() { + hdrGzip = mustDecodeHex(hexHdrGzip) + hdrBzip2 = mustDecodeHex(hexHdrBzip2) + hdrXz = mustDecodeHex(hexHdrXz) + sigTar = mustDecodeHex(hexSigTar) + tarEnd = tarOffset + len(sigTar) +} + +// DetectFileType attempts to detect the type of file that the given reader +// represents by comparing it against known file signatures (magic numbers) +func DetectFileType(r io.Reader) (FileType, error) { + var b bytes.Buffer + n, err := io.CopyN(&b, r, readLen) + if err != nil && err != io.EOF { + return TypeUnknown, err + } + bs := b.Bytes() + switch { + case bytes.HasPrefix(bs, hdrGzip): + return TypeGzip, nil + case bytes.HasPrefix(bs, hdrBzip2): + return TypeBzip2, nil + case bytes.HasPrefix(bs, hdrXz): + return TypeXz, nil + case n > int64(tarEnd) && bytes.Equal(bs[tarOffset:tarEnd], sigTar): + return TypeTar, nil + case http.DetectContentType(bs) == textMime: + return TypeText, nil + default: + return TypeUnknown, nil + } +} + +// XzReader shells out to a command line xz executable (if +// available) to decompress the given io.Reader using the xz +// compression format +func XzReader(r io.Reader) io.ReadCloser { + rpipe, wpipe := io.Pipe() + ex, err := exec.LookPath("xz") + if err != nil { + log.Fatalf("couldn't find xz executable: %v", err) + } + cmd := exec.Command(ex, "--decompress", "--stdout") + cmd.Stdin = r + cmd.Stdout = wpipe + + go func() { + err := cmd.Run() + wpipe.CloseWithError(err) + }() + + return rpipe +} + +// ManifestFromImage extracts a new schema.ImageManifest from the given ACI image. +func ManifestFromImage(rs io.ReadSeeker) (*schema.ImageManifest, error) { + var im schema.ImageManifest + + tr, err := NewCompressedTarReader(rs) + if err != nil { + return nil, err + } + + for { + hdr, err := tr.Next() + switch err { + case io.EOF: + return nil, errors.New("missing manifest") + case nil: + if filepath.Clean(hdr.Name) == ManifestFile { + data, err := ioutil.ReadAll(tr) + if err != nil { + return nil, err + } + if err := im.UnmarshalJSON(data); err != nil { + return nil, err + } + return &im, nil + } + default: + return nil, fmt.Errorf("error extracting tarball: %v", err) + } + } +} + +// NewCompressedTarReader creates a new tar.Reader reading from the given ACI image. +func NewCompressedTarReader(rs io.ReadSeeker) (*tar.Reader, error) { + cr, err := NewCompressedReader(rs) + if err != nil { + return nil, err + } + return tar.NewReader(cr), nil +} + +// NewCompressedReader creates a new io.Reader from the given ACI image. +func NewCompressedReader(rs io.ReadSeeker) (io.Reader, error) { + + var ( + dr io.Reader + err error + ) + + _, err = rs.Seek(0, 0) + if err != nil { + return nil, err + } + + ftype, err := DetectFileType(rs) + if err != nil { + return nil, err + } + + _, err = rs.Seek(0, 0) + if err != nil { + return nil, err + } + + switch ftype { + case TypeGzip: + dr, err = gzip.NewReader(rs) + if err != nil { + return nil, err + } + case TypeBzip2: + dr = bzip2.NewReader(rs) + case TypeXz: + dr = XzReader(rs) + case TypeTar: + dr = rs + case TypeUnknown: + return nil, errors.New("error: unknown image filetype") + default: + return nil, errors.New("no type returned from DetectFileType?") + } + return dr, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/file_test.go b/Godeps/_workspace/src/github.com/appc/spec/aci/file_test.go new file mode 100644 index 00000000000..f7def459966 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/file_test.go @@ -0,0 +1,136 @@ +package aci + +import ( + "archive/tar" + "compress/gzip" + "io/ioutil" + "os" + "testing" +) + +func newTestACI(usedotslash bool) (*os.File, error) { + tf, err := ioutil.TempFile("", "") + if err != nil { + return nil, err + } + + manifestBody := `{"acKind":"ImageManifest","acVersion":"0.5.1","name":"example.com/app"}` + + gw := gzip.NewWriter(tf) + tw := tar.NewWriter(gw) + + manifestPath := "manifest" + if usedotslash { + manifestPath = "./" + manifestPath + } + hdr := &tar.Header{ + Name: manifestPath, + Size: int64(len(manifestBody)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(manifestBody)); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + if err := gw.Close(); err != nil { + return nil, err + } + return tf, nil +} + +func newEmptyTestACI() (*os.File, error) { + tf, err := ioutil.TempFile("", "") + if err != nil { + return nil, err + } + gw := gzip.NewWriter(tf) + tw := tar.NewWriter(gw) + if err := tw.Close(); err != nil { + return nil, err + } + if err := gw.Close(); err != nil { + return nil, err + } + return tf, nil +} + +func TestManifestFromImage(t *testing.T) { + for _, usedotslash := range []bool{false, true} { + img, err := newTestACI(usedotslash) + if err != nil { + t.Fatalf("newTestACI: unexpected error: %v", err) + } + defer img.Close() + defer os.Remove(img.Name()) + + im, err := ManifestFromImage(img) + if err != nil { + t.Fatalf("ManifestFromImage: unexpected error: %v", err) + } + if im.Name.String() != "example.com/app" { + t.Errorf("expected %s, got %s", "example.com/app", im.Name.String()) + } + + emptyImg, err := newEmptyTestACI() + if err != nil { + t.Fatalf("newEmptyTestACI: unexpected error: %v", err) + } + defer emptyImg.Close() + defer os.Remove(emptyImg.Name()) + + im, err = ManifestFromImage(emptyImg) + if err == nil { + t.Fatalf("ManifestFromImage: expected error") + } + } +} + +func TestNewCompressedTarReader(t *testing.T) { + img, err := newTestACI(false) + if err != nil { + t.Fatalf("newTestACI: unexpected error: %v", err) + } + defer img.Close() + defer os.Remove(img.Name()) + + cr, err := NewCompressedTarReader(img) + if err != nil { + t.Fatalf("NewCompressedTarReader: unexpected error: %v", err) + } + + ftype, err := DetectFileType(cr) + if err != nil { + t.Fatalf("DetectFileType: unexpected error: %v", err) + } + + if ftype != TypeText { + t.Errorf("expected %v, got %v", TypeText, ftype) + } +} + +func TestNewCompressedReader(t *testing.T) { + img, err := newTestACI(false) + if err != nil { + t.Fatalf("newTestACI: unexpected error: %v", err) + } + defer img.Close() + defer os.Remove(img.Name()) + + cr, err := NewCompressedReader(img) + if err != nil { + t.Fatalf("NewCompressedReader: unexpected error: %v", err) + } + + ftype, err := DetectFileType(cr) + if err != nil { + t.Fatalf("DetectFileType: unexpected error: %v", err) + } + + if ftype != TypeTar { + t.Errorf("expected %v, got %v", TypeTar, ftype) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/layout.go b/Godeps/_workspace/src/github.com/appc/spec/aci/layout.go new file mode 100644 index 00000000000..03e6a93caab --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/layout.go @@ -0,0 +1,159 @@ +package aci + +/* + +Image Layout + +The on-disk layout of an app container is straightforward. +It includes a rootfs with all of the files that will exist in the root of the app and a manifest describing the image. +The layout MUST contain an image manifest. + +/manifest +/rootfs/ +/rootfs/usr/bin/mysql + +*/ + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/appc/spec/schema" +) + +const ( + // Path to manifest file inside the layout + ManifestFile = "manifest" + // Path to rootfs directory inside the layout + RootfsDir = "rootfs" +) + +var ( + ErrNoRootFS = errors.New("no rootfs found in layout") + ErrNoManifest = errors.New("no image manifest found in layout") +) + +// ValidateLayout takes a directory and validates that the layout of the directory +// matches that expected by the Application Container Image format. +// If any errors are encountered during the validation, it will abort and +// return the first one. +func ValidateLayout(dir string) error { + fi, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("error accessing layout: %v", err) + } + if !fi.IsDir() { + return fmt.Errorf("given path %q is not a directory", dir) + } + var flist []string + var imOK, rfsOK bool + var im io.Reader + walkLayout := func(fpath string, fi os.FileInfo, err error) error { + rpath, err := filepath.Rel(dir, fpath) + if err != nil { + return err + } + switch rpath { + case ".": + case ManifestFile: + im, err = os.Open(fpath) + if err != nil { + return err + } + imOK = true + case RootfsDir: + if !fi.IsDir() { + return errors.New("rootfs is not a directory") + } + rfsOK = true + default: + flist = append(flist, rpath) + } + return nil + } + if err := filepath.Walk(dir, walkLayout); err != nil { + return err + } + return validate(imOK, im, rfsOK, flist) +} + +// ValidateArchive takes a *tar.Reader and validates that the layout of the +// filesystem the reader encapsulates matches that expected by the +// Application Container Image format. If any errors are encountered during +// the validation, it will abort and return the first one. +func ValidateArchive(tr *tar.Reader) error { + var fseen map[string]bool = make(map[string]bool) + var imOK, rfsOK bool + var im bytes.Buffer +Tar: + for { + hdr, err := tr.Next() + switch { + case err == nil: + case err == io.EOF: + break Tar + default: + return err + } + name := filepath.Clean(hdr.Name) + switch name { + case ".": + case ManifestFile: + _, err := io.Copy(&im, tr) + if err != nil { + return err + } + imOK = true + case RootfsDir: + if !hdr.FileInfo().IsDir() { + return fmt.Errorf("rootfs is not a directory") + } + rfsOK = true + default: + if _, seen := fseen[name]; seen { + return fmt.Errorf("duplicate file entry in archive: %s", name) + } + fseen[name] = true + } + } + var flist []string + for key := range fseen { + flist = append(flist, key) + } + return validate(imOK, &im, rfsOK, flist) +} + +func validate(imOK bool, im io.Reader, rfsOK bool, files []string) error { + defer func() { + if rc, ok := im.(io.Closer); ok { + rc.Close() + } + }() + if !imOK { + return ErrNoManifest + } + if !rfsOK { + return ErrNoRootFS + } + b, err := ioutil.ReadAll(im) + if err != nil { + return fmt.Errorf("error reading image manifest: %v", err) + } + var a schema.ImageManifest + if err := a.UnmarshalJSON(b); err != nil { + return fmt.Errorf("image manifest validation failed: %v", err) + } + for _, f := range files { + if !strings.HasPrefix(f, "rootfs") { + return fmt.Errorf("unrecognized file path in layout: %q", f) + } + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/layout_test.go b/Godeps/_workspace/src/github.com/appc/spec/aci/layout_test.go new file mode 100644 index 00000000000..8f28414b890 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/layout_test.go @@ -0,0 +1,62 @@ +package aci + +import ( + "io/ioutil" + "os" + "path" + "testing" +) + +func newValidateLayoutTest() (string, error) { + td, err := ioutil.TempDir("", "") + if err != nil { + return "", err + } + + if err := os.MkdirAll(path.Join(td, "rootfs"), 0755); err != nil { + return "", err + } + + if err := os.MkdirAll(path.Join(td, "rootfs", "dir", "rootfs"), 0755); err != nil { + return "", err + } + + evilManifestBody := "malformedManifest" + manifestBody := `{"acKind":"ImageManifest","acVersion":"0.3.0","name":"example.com/app"}` + + evilManifestPath := "rootfs/manifest" + evilManifestPath = path.Join(td, evilManifestPath) + + em, err := os.Create(evilManifestPath) + if err != nil { + return "", err + } + + em.WriteString(evilManifestBody) + em.Close() + + manifestPath := path.Join(td, "manifest") + + m, err := os.Create(manifestPath) + if err != nil { + return "", err + } + + m.WriteString(manifestBody) + m.Close() + + return td, nil +} + +func TestValidateLayout(t *testing.T) { + layoutPath, err := newValidateLayoutTest() + if err != nil { + t.Fatalf("newValidateLayoutTest: unexpected error: %v", err) + } + defer os.RemoveAll(layoutPath) + + err = ValidateLayout(layoutPath) + if err != nil { + t.Fatalf("ValidateLayout: unexpected error: %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/writer.go b/Godeps/_workspace/src/github.com/appc/spec/aci/writer.go new file mode 100644 index 00000000000..89833574460 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/writer.go @@ -0,0 +1,84 @@ +package aci + +import ( + "archive/tar" + "bytes" + "encoding/json" + "io" + "time" + + "github.com/appc/spec/schema" +) + +// ArchiveWriter writes App Container Images. Users wanting to create an ACI or +// should create an ArchiveWriter and add files to it; the ACI will be written +// to the underlying tar.Writer +type ArchiveWriter interface { + AddFile(hdr *tar.Header, r io.Reader) error + Close() error +} + +type imageArchiveWriter struct { + *tar.Writer + am *schema.ImageManifest +} + +// NewImageWriter creates a new ArchiveWriter which will generate an App +// Container Image based on the given manifest and write it to the given +// tar.Writer +func NewImageWriter(am schema.ImageManifest, w *tar.Writer) ArchiveWriter { + aw := &imageArchiveWriter{ + w, + &am, + } + return aw +} + +func (aw *imageArchiveWriter) AddFile(hdr *tar.Header, r io.Reader) error { + err := aw.Writer.WriteHeader(hdr) + if err != nil { + return err + } + + if r != nil { + _, err := io.Copy(aw.Writer, r) + if err != nil { + return err + } + } + + return nil +} + +func (aw *imageArchiveWriter) addFileNow(path string, contents []byte) error { + buf := bytes.NewBuffer(contents) + now := time.Now() + hdr := tar.Header{ + Name: path, + Mode: 0644, + Uid: 0, + Gid: 0, + Size: int64(buf.Len()), + ModTime: now, + Typeflag: tar.TypeReg, + Uname: "root", + Gname: "root", + ChangeTime: now, + } + return aw.AddFile(&hdr, buf) +} + +func (aw *imageArchiveWriter) addManifest(name string, m json.Marshaler) error { + out, err := m.MarshalJSON() + if err != nil { + return err + } + return aw.addFileNow(name, out) +} + +func (aw *imageArchiveWriter) Close() error { + if err := aw.addManifest(ManifestFile, aw.am); err != nil { + return err + } + return aw.Writer.Close() +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/acirenderer.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/acirenderer.go new file mode 100644 index 00000000000..63f83e176b3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/acirenderer.go @@ -0,0 +1,223 @@ +package acirenderer + +import ( + "archive/tar" + "crypto/sha512" + "fmt" + "hash" + "io" + "io/ioutil" + "path/filepath" + "strings" + + "github.com/appc/spec/schema" + "github.com/appc/spec/schema/types" +) + +// An ACIRegistry provides all functions of an ACIProvider plus functions to +// search for an aci and get its contents +type ACIRegistry interface { + ACIProvider + GetImageManifest(key string) (*schema.ImageManifest, error) + GetACI(name types.ACName, labels types.Labels) (string, error) +} + +// An ACIProvider provides functions to get an ACI contents, to convert an +// ACI hash to the key under which the ACI is known to the provider and to resolve an +// ImageID to the key under which it's known to the provider. +type ACIProvider interface { + // Read the ACI contents stream given the key. Use ResolveKey to + // convert an ImageID to the relative provider's key. + ReadStream(key string) (io.ReadCloser, error) + // Converts an ImageID to the, if existent, key under which the + // ACI is known to the provider + ResolveKey(key string) (string, error) + // Converts a Hash to the provider's key + HashToKey(h hash.Hash) string +} + +// An Image contains the ImageManifest, the ACIProvider's key and its Level in +// the dependency tree. +type Image struct { + Im *schema.ImageManifest + Key string + Level uint16 +} + +// Images encapsulates an ordered slice of Image structs. It represents a flat +// dependency tree. +// The upper Image should be the first in the slice with a level of 0. +// For example if A is the upper image and has two deps (in order B and C). And C has one dep (D), +// the slice (reporting the app name and excluding im and Hash) should be: +// [{A, Level: 0}, {C, Level:1}, {D, Level: 2}, {B, Level: 1}] +type Images []Image + +// ACIFiles represents which files to extract for every ACI +type ACIFiles struct { + Key string + FileMap map[string]struct{} +} + +// RenderedACI is an (ordered) slice of ACIFiles +type RenderedACI []*ACIFiles + +// GetRenderedACIWithImageID, given an imageID, starts with the matching image +// available in the store, creates the dependencies list and returns the +// RenderedACI list. +func GetRenderedACIWithImageID(imageID types.Hash, ap ACIRegistry) (RenderedACI, error) { + imgs, err := CreateDepListFromImageID(imageID, ap) + if err != nil { + return nil, err + } + return GetRenderedACIFromList(imgs, ap) +} + +// GetRenderedACI, given an image app name and optional labels, starts with the +// best matching image available in the store, creates the dependencies list +// and returns the RenderedACI list. +func GetRenderedACI(name types.ACName, labels types.Labels, ap ACIRegistry) (RenderedACI, error) { + imgs, err := CreateDepListFromNameLabels(name, labels, ap) + if err != nil { + return nil, err + } + return GetRenderedACIFromList(imgs, ap) +} + +// GetRenderedACIFromList returns the RenderedACI list. All file outside rootfs +// are excluded (at the moment only "manifest"). +func GetRenderedACIFromList(imgs Images, ap ACIProvider) (RenderedACI, error) { + if len(imgs) == 0 { + return nil, fmt.Errorf("image list empty") + } + + allFiles := make(map[string]struct{}) + renderedACI := RenderedACI{} + + first := true + for i, img := range imgs { + pwlm := getUpperPWLM(imgs, i) + ra, err := getACIFiles(img, ap, allFiles, pwlm) + if err != nil { + return nil, err + } + // Use the manifest from the upper ACI + if first { + ra.FileMap["manifest"] = struct{}{} + first = false + } + renderedACI = append(renderedACI, ra) + } + + return renderedACI, nil +} + +// getUpperPWLM returns the pwl at the lower level for the branch where +// img[pos] lives. +func getUpperPWLM(imgs Images, pos int) map[string]struct{} { + var pwlm map[string]struct{} + curlevel := imgs[pos].Level + // Start from our position and go back ignoring the other leafs. + for i := pos; i >= 0; i-- { + img := imgs[i] + if img.Level < curlevel && len(img.Im.PathWhitelist) > 0 { + pwlm = pwlToMap(img.Im.PathWhitelist) + } + curlevel = img.Level + } + return pwlm +} + +// getACIFiles returns the ACIFiles struct for the given image. All files +// outside rootfs are excluded (at the moment only "manifest"). +func getACIFiles(img Image, ap ACIProvider, allFiles map[string]struct{}, pwlm map[string]struct{}) (*ACIFiles, error) { + rs, err := ap.ReadStream(img.Key) + if err != nil { + return nil, err + } + defer rs.Close() + + hash := sha512.New() + r := io.TeeReader(rs, hash) + + thispwlm := pwlToMap(img.Im.PathWhitelist) + ra := &ACIFiles{FileMap: make(map[string]struct{})} + if err = Walk(tar.NewReader(r), func(hdr *tar.Header) error { + name := hdr.Name + cleanName := filepath.Clean(name) + + // Ignore files outside /rootfs/ (at the moment only "manifest") + if !strings.HasPrefix(cleanName, "rootfs/") { + return nil + } + + // Is the file in our PathWhiteList? + // If the file is a directory continue also if not in PathWhiteList + if hdr.Typeflag != tar.TypeDir { + if len(img.Im.PathWhitelist) > 0 { + if _, ok := thispwlm[cleanName]; !ok { + return nil + } + } + } + // Is the file in the lower level PathWhiteList of this img branch? + if pwlm != nil { + if _, ok := pwlm[cleanName]; !ok { + return nil + } + } + // Is the file already provided by a previous image? + if _, ok := allFiles[cleanName]; ok { + return nil + } + ra.FileMap[cleanName] = struct{}{} + allFiles[cleanName] = struct{}{} + return nil + }); err != nil { + return nil, err + } + + // Tar does not necessarily read the complete file, so ensure we read the entirety into the hash + if _, err := io.Copy(ioutil.Discard, r); err != nil { + return nil, fmt.Errorf("error reading ACI: %v", err) + } + + if g := ap.HashToKey(hash); g != img.Key { + return nil, fmt.Errorf("image hash does not match expected (%s != %s)", g, img.Key) + } + + ra.Key = img.Key + return ra, nil +} + +// pwlToMap converts a pathWhiteList slice to a map for faster search +// It will also prepend "rootfs/" to the provided paths and they will be +// relative to "/" so they can be easily compared with the tar.Header.Name +// If pwl length is 0, a nil map is returned +func pwlToMap(pwl []string) map[string]struct{} { + if len(pwl) == 0 { + return nil + } + m := make(map[string]struct{}, len(pwl)) + for _, name := range pwl { + relpath := filepath.Join("rootfs", name) + m[relpath] = struct{}{} + } + return m +} + +func Walk(tarReader *tar.Reader, walkFunc func(hdr *tar.Header) error) error { + for { + hdr, err := tarReader.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return fmt.Errorf("Error reading tar entry: %v", err) + } + if err := walkFunc(hdr); err != nil { + return err + } + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/acirenderer_test.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/acirenderer_test.go new file mode 100644 index 00000000000..8185d1e31d0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/acirenderer_test.go @@ -0,0 +1,1998 @@ +package acirenderer + +import ( + "archive/tar" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/appc/spec/schema" + "github.com/appc/spec/schema/types" +) + +const tstprefix = "acirenderer-test" + +type testTarEntry struct { + header *tar.Header + contents string +} + +func newTestTar(entries []*testTarEntry, dir string) (string, error) { + t, err := ioutil.TempFile(dir, "tar") + if err != nil { + return "", err + } + defer t.Close() + tw := tar.NewWriter(t) + for _, entry := range entries { + // Add default mode + if entry.header.Mode == 0 { + if entry.header.Typeflag == tar.TypeDir { + entry.header.Mode = 0755 + } else { + entry.header.Mode = 0644 + } + } + // Add calling user uid and gid or tests will fail + entry.header.Uid = os.Getuid() + entry.header.Gid = os.Getgid() + if err := tw.WriteHeader(entry.header); err != nil { + return "", err + } + if _, err := io.WriteString(tw, entry.contents); err != nil { + return "", err + } + } + if err := tw.Close(); err != nil { + return "", err + } + return t.Name(), nil +} + +type fileInfo struct { + path string + typeflag byte + size int64 + mode os.FileMode +} + +func newTestACI(entries []*testTarEntry, dir string, ds *TestStore) (string, error) { + testTarPath, err := newTestTar(entries, dir) + if err != nil { + return "", err + } + + key, err := ds.WriteACI(testTarPath) + if err != nil { + return "", err + } + + return key, nil +} + +func createImageManifest(imj string) (*schema.ImageManifest, error) { + var im schema.ImageManifest + err := im.UnmarshalJSON([]byte(imj)) + if err != nil { + return nil, err + } + return &im, nil +} + +func addDependencies(imj string, deps ...types.Dependency) (string, error) { + im, err := createImageManifest(imj) + if err != nil { + return "", err + } + + for _, dep := range deps { + im.Dependencies = append(im.Dependencies, dep) + } + imjb, err := im.MarshalJSON() + return string(imjb), err +} + +func genSimpleImage(imj string, pwl []string, level uint16, dir string, ds *TestStore) (*Image, error) { + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + } + key, err := newTestACI(entries, dir, ds) + if err != nil { + return nil, fmt.Errorf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + return nil, fmt.Errorf("unexpected error: %v", err) + } + im.PathWhitelist = pwl + image1 := &Image{Im: im, Key: key, Level: level} + return image1, nil + +} + +func TestGetUpperPWLM(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + l0pwl1 := []string{"/a/path/white/list/level0/1"} + l1pwl1 := []string{"/a/path/white/list/level1/1"} + + l0pwl1m := pwlToMap(l0pwl1) + + // An image at level 0 with l0pwl1 + iml0pwl1, err := genSimpleImage(imj, l0pwl1, 0, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // An image at level 0 without pwl + iml0nopwl, err := genSimpleImage(imj, []string{}, 0, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // An image at level 1 with l1pwl1 + iml1pwl1, err := genSimpleImage(imj, l1pwl1, 1, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // An image at level 1 without pwl + iml1nopwl, err := genSimpleImage(imj, []string{}, 0, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // An image at level 2 without pwl + iml2nopwl, err := genSimpleImage(imj, []string{}, 2, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var emptypwlm map[string]struct{} + + // A (pwl) + // Searching for the upper pwlm of A should return nil + A := *iml0pwl1 + images := Images{A} + pwlm := getUpperPWLM(images, 0) + if !reflect.DeepEqual(pwlm, emptypwlm) { + t.Errorf("wrong PathWhitelist, got %#v, want: %#v", pwlm, emptypwlm) + + } + + // A (pwl) ---- B (pwl) --- C + // Searching for the upper pwlm of C should return l0pwl1m + A = *iml0pwl1 + B := *iml1pwl1 + C := *iml2nopwl + images = Images{A, B, C} + pwlm = getUpperPWLM(images, 2) + if !reflect.DeepEqual(pwlm, l0pwl1m) { + t.Errorf("wrong PathWhitelist, got %#v, want: %#v", pwlm, l0pwl1m) + + } + + // A ---- B --- D + // \-- C (pwl) + // Searching for the upper pwlm of C should return nil + A = *iml0nopwl + B = *iml1nopwl + C = *iml1pwl1 + D := *iml2nopwl + images = Images{A, C, B, D} + pwlm = getUpperPWLM(images, 3) + if !reflect.DeepEqual(pwlm, emptypwlm) { + t.Errorf("wrong PathWhitelist, got %#v, want: %#v", pwlm, emptypwlm) + } +} + +// Test an image with 1 dep. The parent provides a dir not provided by the image. +func TestDirFromParent(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // An empty dir + { + header: &tar.Header{ + Name: "rootfs/a", + Typeflag: tar.TypeDir, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a", typeflag: tar.TypeDir}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 0} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with 1 dep. The image provides a dir not provided by the parent. +func TestNewDir(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // An empty dir + { + header: &tar.Header{ + Name: "rootfs/a", + Typeflag: tar.TypeDir, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 0} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a", typeflag: tar.TypeDir}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 1} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with 1 dep. The image overrides dirs modes from the parent dep. Verifies the right permissions. +func TestDirOverride(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + header: &tar.Header{ + Name: "rootfs/a", + Typeflag: tar.TypeDir, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // An empty dir + { + header: &tar.Header{ + Name: "rootfs/a", + Typeflag: tar.TypeDir, + Mode: 0700, + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a", typeflag: tar.TypeDir, mode: 0700}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 0} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with 1 dep. The parent provides a file not provided by the image. +func TestFileFromParent(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 5, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 0} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 5}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 1} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with 1 dep. The image provides a file not provided by the parent. +func TestNewFile(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 0} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 10, + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 10}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 1} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with 1 dep. The image overrides a file already provided by the parent dep. +func TestFileOverride(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 5, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 10, + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 10}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 0} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with 1 dep. The parent image has a pathWhiteList. +func TestPWLOnlyParent(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01", + "pathWhitelist" : [ "/a/file01.txt", "/a/file02.txt", "/b/link01.txt", "/c/", "/d/" ] + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 5, + }, + }, + // This should not appear in rendered aci + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file03.txt", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "rootfs/b/link01.txt", + Linkname: "file01.txt", + Typeflag: tar.TypeSymlink, + }, + }, + // The file "rootfs/c/file01.txt" should not appear but a new file "rootfs/c/file02.txt" provided by the upper image should appear. + // The directory should be left with its permissions + { + header: &tar.Header{ + Name: "rootfs/c", + Typeflag: tar.TypeDir, + Mode: 0700, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/c/file01.txt", + Size: 5, + Mode: 0700, + }, + }, + // The file "rootfs/d/file01.txt" should not appear but the directory should be left and also its permissions + { + header: &tar.Header{ + Name: "rootfs/d", + Typeflag: tar.TypeDir, + Mode: 0700, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/d/file01.txt", + Size: 5, + Mode: 0700, + }, + }, + // The file and the directory should not appear + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/e/file01.txt", + Size: 5, + Mode: 0700, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/b/file01.txt", + Size: 10, + }, + }, + // New file + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/c/file02.txt", + Size: 5, + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/a/file02.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/link01.txt", typeflag: tar.TypeSymlink}, + &fileInfo{path: "rootfs/b/file01.txt", typeflag: tar.TypeReg, size: 10}, + &fileInfo{path: "rootfs/c", typeflag: tar.TypeDir, mode: 0700}, + &fileInfo{path: "rootfs/c/file02.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/d", typeflag: tar.TypeDir, mode: 0700}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 0} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with 1 dep. The upper image has a pathWhiteList. +func TestPWLOnlyImage(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image in rendered aci + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 5, + }, + }, + // It should not appear in rendered aci + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file03.txt", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "rootfs/b/link01.txt", + Linkname: "file01.txt", + Typeflag: tar.TypeSymlink, + }, + }, + // The file "rootfs/c/file01.txt" should not appear but a new file "rootfs/c/file02.txt" provided by the upper image should appear. + { + header: &tar.Header{ + Name: "rootfs/c", + Typeflag: tar.TypeDir, + Mode: 0700, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/c/file01.txt", + Size: 5, + Mode: 0700, + }, + }, + // The file "rootfs/d/file01.txt" should not appear but the directory should be left and also its permissions + { + header: &tar.Header{ + Name: "rootfs/d", + Typeflag: tar.TypeDir, + Mode: 0700, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/d/file01.txt", + Size: 5, + Mode: 0700, + }, + }, + // The file and the directory should not appear + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/e/file01.txt", + Size: 5, + Mode: 0700, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02", + "pathWhitelist" : [ "/a/file01.txt", "/a/file02.txt", "/b/link01.txt", "/b/file01.txt", "/c/file02.txt", "/d/" ] + } + ` + + k1, _ := types.NewHash(key1) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 10, + }, + }, + // New file + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/c/file02.txt", + Size: 5, + }, + }, + } + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 10}, + &fileInfo{path: "rootfs/a/file02.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/link01.txt", typeflag: tar.TypeSymlink}, + &fileInfo{path: "rootfs/c/file02.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/d", typeflag: tar.TypeDir, mode: 0700}, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 0} + + images := Images{image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test02", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with a pathwhitelist and 2 deps (first with pathWhiteList and the second without pathWhiteList) +// A (pwl) ---- B (pwl) +// \-- C +func Test2Deps1(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01", + "pathWhitelist" : [ "/a/file01.txt", "/a/file02.txt", "/a/file03.txt", "/a/file04.txt", "/b/link01.txt", "/b/file01.txt" ] + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 5, + }, + }, + // It should be overridden by the one provided by the next dep + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 5, + }, + }, + // It should remain like this + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file03.txt", + Size: 5, + }, + }, + // It should not appear in rendered aci + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file04.txt", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "rootfs/b/link01.txt", + Linkname: "file01.txt", + Typeflag: tar.TypeSymlink, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 10, + }, + }, + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 10, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/b/file01.txt", + Size: 5, + }, + }, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test03", + "pathWhitelist" : [ "/a/file01.txt", "/a/file02.txt", "/a/file03.txt", "/b/link01.txt", "/b/file01.txt", "/b/file02.txt", "/c/file01.txt" ] + } + ` + + k1, _ := types.NewHash(key1) + k2, _ := types.NewHash(key2) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + types.Dependency{ + App: "example.com/test02", + ImageID: k2}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // Overridden + { + contents: "hellohellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 15, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/b/file02.txt", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/c/file01.txt", + Size: 5, + }, + }, + } + + key3, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image3 := Image{Im: im, Key: key3, Level: 0} + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 15}, + &fileInfo{path: "rootfs/a/file02.txt", typeflag: tar.TypeReg, size: 10}, + &fileInfo{path: "rootfs/a/file03.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/link01.txt", typeflag: tar.TypeSymlink}, + &fileInfo{path: "rootfs/b/file01.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/file02.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/c/file01.txt", typeflag: tar.TypeReg, size: 5}, + } + + images := Images{image3, image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test03", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test an image with a pathwhitelist and 2 deps (first without pathWhiteList and the second with pathWhiteList) +// A (pwl) ---- B +// \-- C (pwl) +func Test2Deps2(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 5, + }, + }, + // It should be overridden by the one provided by the next dep + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 5, + }, + }, + // It should remain like this + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file03.txt", + Size: 5, + }, + }, + // It should not appear in rendered aci + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file04.txt", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "rootfs/b/link01.txt", + Linkname: "file01.txt", + Typeflag: tar.TypeSymlink, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02", + "pathWhitelist" : [ "/a/file01.txt", "/a/file02.txt", "/b/file01.txt" ] + } + ` + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 10, + }, + }, + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 10, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/b/file01.txt", + Size: 5, + }, + }, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 1} + + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test03", + "pathWhitelist" : [ "/a/file01.txt", "/a/file02.txt", "/a/file03.txt", "/b/link01.txt", "/b/file01.txt", "/b/file02.txt", "/c/file01.txt" ] + } + ` + + k1, _ := types.NewHash(key1) + k2, _ := types.NewHash(key2) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + types.Dependency{ + App: "example.com/test02", + ImageID: k2}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // Overridden + { + contents: "hellohellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 15, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/b/file02.txt", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/c/file01.txt", + Size: 5, + }, + }, + } + + key3, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image3 := Image{Im: im, Key: key3, Level: 0} + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 15}, + &fileInfo{path: "rootfs/a/file02.txt", typeflag: tar.TypeReg, size: 10}, + &fileInfo{path: "rootfs/a/file03.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/link01.txt", typeflag: tar.TypeSymlink}, + &fileInfo{path: "rootfs/b/file01.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/file02.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/c/file01.txt", typeflag: tar.TypeReg, size: 5}, + } + + images := Images{image3, image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test03", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Test A (pwl) ---- B +// \-- C -- D +func Test3Deps(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + ds := NewTestStore() + + // B + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + } + ` + + entries := []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 5, + }, + }, + // It should be overridden by the one provided by the next dep + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 5, + }, + }, + // It should remain like this + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file03.txt", + Size: 5, + }, + }, + // It should not appear in rendered aci + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file04.txt", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "rootfs/b/link01.txt", + Linkname: "file01.txt", + Typeflag: tar.TypeSymlink, + }, + }, + } + + key1, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err := createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image1 := Image{Im: im, Key: key1, Level: 1} + + // D + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test03" + } + ` + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/b/file01.txt", + Size: 5, + }, + }, + // It should not appear in rendered aci + { + header: &tar.Header{ + Name: "rootfs/d", + Typeflag: tar.TypeDir, + Mode: 0700, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/d/file01.txt", + Size: 5, + Mode: 0700, + }, + }, + } + + key2, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image2 := Image{Im: im, Key: key2, Level: 2} + + // C + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02" + } + ` + k2, _ := types.NewHash(key2) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test03", + ImageID: k2}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // It should be overridden by the one provided by the upper image + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 10, + }, + }, + { + contents: "hellohello", + header: &tar.Header{ + Name: "rootfs/a/file02.txt", + Size: 10, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/b/file01.txt", + Size: 5, + }, + }, + } + + key3, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image3 := Image{Im: im, Key: key3, Level: 1} + + // A + imj = ` + { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test04", + "pathWhitelist" : [ "/a/file01.txt", "/a/file02.txt", "/a/file03.txt", "/b/link01.txt", "/b/file01.txt", "/b/file02.txt", "/c/file01.txt" ] + } + ` + + k1, _ := types.NewHash(key1) + k3, _ := types.NewHash(key3) + imj, err = addDependencies(imj, + types.Dependency{ + App: "example.com/test01", + ImageID: k1}, + types.Dependency{ + App: "example.com/test02", + ImageID: k3}, + ) + + entries = []*testTarEntry{ + { + contents: imj, + header: &tar.Header{ + Name: "manifest", + Size: int64(len(imj)), + }, + }, + // Overridden + { + contents: "hellohellohello", + header: &tar.Header{ + Name: "rootfs/a/file01.txt", + Size: 15, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/b/file02.txt", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "rootfs/c/file01.txt", + Size: 5, + }, + }, + } + + key4, err := newTestACI(entries, dir, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + im, err = createImageManifest(imj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + image4 := Image{Im: im, Key: key4, Level: 0} + + expectedFiles := []*fileInfo{ + &fileInfo{path: "manifest", typeflag: tar.TypeReg}, + &fileInfo{path: "rootfs/a/file01.txt", typeflag: tar.TypeReg, size: 15}, + &fileInfo{path: "rootfs/a/file02.txt", typeflag: tar.TypeReg, size: 10}, + &fileInfo{path: "rootfs/a/file03.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/link01.txt", typeflag: tar.TypeSymlink}, + &fileInfo{path: "rootfs/b/file01.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/b/file02.txt", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "rootfs/c/file01.txt", typeflag: tar.TypeReg, size: 5}, + } + + images := Images{image4, image3, image2, image1} + err = checkRenderACIFromList(images, expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + err = checkRenderACI("example.com/test04", expectedFiles, ds) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// Given an image app name and optional labels, get the best matching image +// available in the store, build its dependency list and render it inside dir +func RenderACI(name types.ACName, labels types.Labels, ap ACIRegistry) (map[string]*fileInfo, error) { + renderedACI, err := GetRenderedACI(name, labels, ap) + if err != nil { + return nil, err + } + return renderImage(renderedACI, ap) +} + +// Given an already populated dependency list, it will extract, under the provided +// directory, the rendered ACI +func RenderACIFromList(imgs Images, ap ACIProvider) (map[string]*fileInfo, error) { + renderedACI, err := GetRenderedACIFromList(imgs, ap) + if err != nil { + return nil, err + } + return renderImage(renderedACI, ap) +} + +// Given a RenderedACI, it will extract, under the provided directory, the +// needed files from the right source ACI. +// The manifest will be extracted from the upper ACI. +// No file overwriting is done as it should usually be called +// providing an empty directory. +func renderImage(renderedACI RenderedACI, ap ACIProvider) (map[string]*fileInfo, error) { + files := make(map[string]*fileInfo) + for _, ra := range renderedACI { + rs, err := ap.ReadStream(ra.Key) + if err != nil { + return nil, err + } + defer rs.Close() + tr := tar.NewReader(rs) + for { + hdr, err := tr.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return nil, fmt.Errorf("Error reading tar entry: %v", err) + } + typ := hdr.Typeflag + cleanName := filepath.Clean(hdr.Name) + if _, ok := ra.FileMap[cleanName]; ok { + switch { + case typ == tar.TypeReg || typ == tar.TypeRegA: + files[cleanName] = &fileInfo{path: cleanName, typeflag: tar.TypeReg, size: hdr.Size, mode: hdr.FileInfo().Mode().Perm()} + case typ == tar.TypeDir: + files[cleanName] = &fileInfo{path: cleanName, typeflag: tar.TypeDir, mode: hdr.FileInfo().Mode().Perm()} + case typ == tar.TypeSymlink: + files[cleanName] = &fileInfo{path: cleanName, typeflag: tar.TypeSymlink, mode: hdr.FileInfo().Mode()} + default: + return nil, fmt.Errorf("wrong type flag: %v\n", typ) + } + } + + } + } + return files, nil +} + +func checkRenderACI(app types.ACName, expectedFiles []*fileInfo, ds *TestStore) error { + files, err := RenderACI(app, nil, ds) + if err != nil { + return err + } + err = checkExpectedFiles(files, FISliceToMap(expectedFiles)) + if err != nil { + return err + } + + return nil +} + +func checkRenderACIFromList(images Images, expectedFiles []*fileInfo, ds *TestStore) error { + files, err := RenderACIFromList(images, ds) + if err != nil { + return err + } + err = checkExpectedFiles(files, FISliceToMap(expectedFiles)) + if err != nil { + return err + } + return nil +} + +func checkExpectedFiles(files map[string]*fileInfo, expectedFiles map[string]*fileInfo) error { + // Set defaults for not specified expected file mode + for _, ef := range expectedFiles { + if ef.mode == 0 { + if ef.typeflag == tar.TypeDir { + ef.mode = 0755 + } else { + ef.mode = 0644 + } + } + } + + for _, ef := range expectedFiles { + _, ok := files[ef.path] + if !ok { + return fmt.Errorf("Expected file \"%s\" not in files", ef.path) + } + + } + + for _, file := range files { + ef, ok := expectedFiles[file.path] + if !ok { + return fmt.Errorf("file \"%s\" not in expectedFiles", file.path) + } + if ef.typeflag != file.typeflag { + return fmt.Errorf("file \"%s\": file type differs: found %d, wanted: %d", file.path, file.typeflag, ef.typeflag) + } + if ef.typeflag == tar.TypeReg && file.path != "manifest" { + if ef.size != file.size { + return fmt.Errorf("file \"%s\": size differs: found %d, wanted: %d", file.path, file.size, ef.size) + } + } + // Check modes but ignore symlinks + if ef.mode != file.mode && ef.typeflag != tar.TypeSymlink { + return fmt.Errorf("file \"%s\": mode differs: found %#o, wanted: %#o", file.path, file.mode, ef.mode) + } + + } + return nil +} + +func FISliceToMap(slice []*fileInfo) map[string]*fileInfo { + fim := make(map[string]*fileInfo, len(slice)) + for _, fi := range slice { + fim[fi.path] = fi + } + return fim +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/resolve.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/resolve.go new file mode 100644 index 00000000000..1cc08d4f20f --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/resolve.go @@ -0,0 +1,74 @@ +package acirenderer + +import ( + "container/list" + + "github.com/appc/spec/schema/types" +) + +// CreateDepListFromImageID returns the flat dependency tree of the image with +// the provided imageID +func CreateDepListFromImageID(imageID types.Hash, ap ACIRegistry) (Images, error) { + key, err := ap.ResolveKey(imageID.String()) + if err != nil { + return nil, err + } + return createDepList(key, ap) +} + +// CreateDepListFromNameLabels returns the flat dependency tree of the image +// with the provided app name and optional labels. +func CreateDepListFromNameLabels(name types.ACName, labels types.Labels, ap ACIRegistry) (Images, error) { + key, err := ap.GetACI(name, labels) + if err != nil { + return nil, err + } + return createDepList(key, ap) +} + +// createDepList returns the flat dependency tree as a list of Image type +func createDepList(key string, ap ACIRegistry) (Images, error) { + imgsl := list.New() + im, err := ap.GetImageManifest(key) + if err != nil { + return nil, err + } + + img := Image{Im: im, Key: key, Level: 0} + imgsl.PushFront(img) + + // Create a flat dependency tree. Use a LinkedList to be able to + // insert elements in the list while working on it. + for el := imgsl.Front(); el != nil; el = el.Next() { + img := el.Value.(Image) + dependencies := img.Im.Dependencies + for _, d := range dependencies { + var depimg Image + var depKey string + if d.ImageID != nil && !d.ImageID.Empty() { + depKey, err = ap.ResolveKey(d.ImageID.String()) + if err != nil { + return nil, err + } + } else { + var err error + depKey, err = ap.GetACI(d.App, d.Labels) + if err != nil { + return nil, err + } + } + im, err := ap.GetImageManifest(depKey) + if err != nil { + return nil, err + } + depimg = Image{Im: im, Key: depKey, Level: img.Level + 1} + imgsl.InsertAfter(depimg, el) + } + } + + imgs := Images{} + for el := imgsl.Front(); el != nil; el = el.Next() { + imgs = append(imgs, el.Value.(Image)) + } + return imgs, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/store_test.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/store_test.go new file mode 100644 index 00000000000..c8085588f81 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/acirenderer/store_test.go @@ -0,0 +1,91 @@ +package acirenderer + +import ( + "bytes" + "fmt" + "hash" + "io" + "io/ioutil" + "os" + + "github.com/appc/spec/aci" + "github.com/appc/spec/schema" + "github.com/appc/spec/schema/types" +) + +const ( + hashPrefix = "sha512-" +) + +type TestStoreAci struct { + data []byte + key string + ImageManifest *schema.ImageManifest +} + +type TestStore struct { + acis map[string]*TestStoreAci +} + +func NewTestStore() *TestStore { + return &TestStore{acis: make(map[string]*TestStoreAci)} +} + +func (ts *TestStore) WriteACI(path string) (string, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return "", err + } + imageID := types.NewHashSHA512(data) + + rs, err := os.Open(path) + if err != nil { + return "", err + } + defer rs.Close() + im, err := aci.ManifestFromImage(rs) + if err != nil { + return "", fmt.Errorf("error retrieving ImageManifest: %v", err) + } + + key := imageID.String() + ts.acis[key] = &TestStoreAci{data: data, key: key, ImageManifest: im} + return key, nil +} + +func (ts *TestStore) GetImageManifest(key string) (*schema.ImageManifest, error) { + aci, ok := ts.acis[key] + if !ok { + return nil, fmt.Errorf("aci with key: %s not found", key) + } + return aci.ImageManifest, nil + +} +func (ts *TestStore) GetACI(name types.ACName, labels types.Labels) (string, error) { + for _, aci := range ts.acis { + if aci.ImageManifest.Name.String() == name.String() { + return aci.key, nil + } + } + return "", fmt.Errorf("aci not found") +} + +func (ts *TestStore) ReadStream(key string) (io.ReadCloser, error) { + aci, ok := ts.acis[key] + if !ok { + return nil, fmt.Errorf("stream for key: %s not found", key) + } + return ioutil.NopCloser(bytes.NewReader(aci.data)), nil +} + +func (ts *TestStore) ResolveKey(key string) (string, error) { + return key, nil +} + +// HashToKey takes a hash.Hash (which currently _MUST_ represent a full SHA512), +// calculates its sum, and returns a string which should be used as the key to +// store the data matching the hash. +func (ts *TestStore) HashToKey(h hash.Hash) string { + s := h.Sum(nil) + return fmt.Sprintf("%s%x", hashPrefix, s) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/doc.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/doc.go new file mode 100644 index 00000000000..b5aa8b5296e --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/doc.go @@ -0,0 +1,3 @@ +// Package tarheader contains a simple abstraction to accurately create +// tar.Headers on different operating systems. +package tarheader diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_darwin.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_darwin.go new file mode 100644 index 00000000000..dd0176a8c5e --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_darwin.go @@ -0,0 +1,25 @@ +//+build darwin + +package tarheader + +import ( + "archive/tar" + "os" + "syscall" + "time" +) + +func init() { + populateHeaderStat = append(populateHeaderStat, populateHeaderCtime) +} + +func populateHeaderCtime(h *tar.Header, fi os.FileInfo, _ map[uint64]string) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return + } + + sec, nsec := st.Ctimespec.Unix() + ctime := time.Unix(sec, nsec) + h.ChangeTime = ctime +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_linux.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_linux.go new file mode 100644 index 00000000000..16c0c7e229c --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_linux.go @@ -0,0 +1,23 @@ +package tarheader + +import ( + "archive/tar" + "os" + "syscall" + "time" +) + +func init() { + populateHeaderStat = append(populateHeaderStat, populateHeaderCtime) +} + +func populateHeaderCtime(h *tar.Header, fi os.FileInfo, _ map[uint64]string) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return + } + + sec, nsec := st.Ctim.Unix() + ctime := time.Unix(sec, nsec) + h.ChangeTime = ctime +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_posix.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_posix.go new file mode 100644 index 00000000000..aadc155aa89 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_posix.go @@ -0,0 +1,51 @@ +package tarheader + +/* +#define _BSD_SOURCE +#define _DEFAULT_SOURCE +#include + +unsigned int +my_major(dev_t dev) +{ + return major(dev); +} + +unsigned int +my_minor(dev_t dev) +{ + return minor(dev); +} + +*/ +import "C" +import ( + "archive/tar" + "os" + "syscall" +) + +func init() { + populateHeaderStat = append(populateHeaderStat, populateHeaderUnix) +} + +func populateHeaderUnix(h *tar.Header, fi os.FileInfo, seen map[uint64]string) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return + } + h.Uid = int(st.Uid) + h.Gid = int(st.Gid) + if st.Mode&syscall.S_IFMT == syscall.S_IFBLK || st.Mode&syscall.S_IFMT == syscall.S_IFCHR { + h.Devminor = int64(C.my_minor(C.dev_t(st.Rdev))) + h.Devmajor = int64(C.my_major(C.dev_t(st.Rdev))) + } + // If we have already seen this inode, generate a hardlink + p, ok := seen[uint64(st.Ino)] + if ok { + h.Linkname = p + h.Typeflag = tar.TypeLink + } else { + seen[uint64(st.Ino)] = h.Name + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_posix_test.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_posix_test.go new file mode 100644 index 00000000000..aecfd55b9e0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/pop_posix_test.go @@ -0,0 +1,61 @@ +package tarheader + +import ( + "archive/tar" + "io/ioutil" + "os" + "path/filepath" + "syscall" + "testing" +) + +// mknod requires privilege ... +func TestHeaderUnixDev(t *testing.T) { + hExpect := tar.Header{ + Name: "./dev/test0", + Size: 0, + Typeflag: tar.TypeBlock, + Devminor: 5, + Devmajor: 233, + } + // make our test block device + var path string + { + var err error + path, err = ioutil.TempDir("", "tarheader-test-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(path) + if err := os.Mkdir(filepath.Join(path, "dev"), os.FileMode(0755)); err != nil { + t.Fatal(err) + } + mode := uint32(hExpect.Mode&07777) | syscall.S_IFBLK + dev := uint32(((hExpect.Devminor & 0xfff00) << 12) | ((hExpect.Devmajor & 0xfff) << 8) | (hExpect.Devminor & 0xff)) + if err := syscall.Mknod(filepath.Join(path, hExpect.Name), mode, int(dev)); err != nil { + if err == syscall.EPERM { + t.Skip("no permission to CAP_MKNOD") + } + t.Fatal(err) + } + } + fi, err := os.Stat(filepath.Join(path, hExpect.Name)) + if err != nil { + t.Fatal(err) + } + + hGot := tar.Header{ + Name: "./dev/test0", + Size: 0, + Typeflag: tar.TypeBlock, + } + + seen := map[uint64]string{} + populateHeaderUnix(&hGot, fi, seen) + if hGot.Devminor != hExpect.Devminor { + t.Errorf("dev minor: got %d, expected %d", hGot.Devminor, hExpect.Devminor) + } + if hGot.Devmajor != hExpect.Devmajor { + t.Errorf("dev major: got %d, expected %d", hGot.Devmajor, hExpect.Devmajor) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/tarheader.go b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/tarheader.go new file mode 100644 index 00000000000..d74ec90dc62 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader/tarheader.go @@ -0,0 +1,14 @@ +package tarheader + +import ( + "archive/tar" + "os" +) + +var populateHeaderStat []func(h *tar.Header, fi os.FileInfo, seen map[uint64]string) + +func Populate(h *tar.Header, fi os.FileInfo, seen map[uint64]string) { + for _, pop := range populateHeaderStat { + pop(h, fi, seen) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/doc.go b/Godeps/_workspace/src/github.com/appc/spec/schema/doc.go new file mode 100644 index 00000000000..8107c40ae7c --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/doc.go @@ -0,0 +1,11 @@ +// Package schema provides definitions for the JSON schema of the different +// manifests in the App Container Specification. The manifests are canonically +// represented in their respective structs: +// - `ImageManifest` +// - `PodManifest` +// +// Validation is performed through serialization: if a blob of JSON data will +// unmarshal to one of the *Manifests, it is considered a valid implementation +// of the standard. Similarly, if a constructed *Manifest struct marshals +// successfully to JSON, it must be valid. +package schema diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/image.go b/Godeps/_workspace/src/github.com/appc/spec/schema/image.go new file mode 100644 index 00000000000..f5c652918b2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/image.go @@ -0,0 +1,81 @@ +package schema + +import ( + "encoding/json" + "errors" + + "github.com/appc/spec/schema/types" +) + +const ( + ACIExtension = ".aci" + ImageManifestKind = types.ACKind("ImageManifest") +) + +type ImageManifest struct { + ACKind types.ACKind `json:"acKind"` + ACVersion types.SemVer `json:"acVersion"` + Name types.ACName `json:"name"` + Labels types.Labels `json:"labels,omitempty"` + App *types.App `json:"app,omitempty"` + Annotations types.Annotations `json:"annotations,omitempty"` + Dependencies types.Dependencies `json:"dependencies,omitempty"` + PathWhitelist []string `json:"pathWhitelist,omitempty"` +} + +// imageManifest is a model to facilitate extra validation during the +// unmarshalling of the ImageManifest +type imageManifest ImageManifest + +func BlankImageManifest() *ImageManifest { + return &ImageManifest{ACKind: ImageManifestKind, ACVersion: AppContainerVersion} +} + +func (im *ImageManifest) UnmarshalJSON(data []byte) error { + a := imageManifest(*im) + err := json.Unmarshal(data, &a) + if err != nil { + return err + } + nim := ImageManifest(a) + if err := nim.assertValid(); err != nil { + return err + } + *im = nim + return nil +} + +func (im ImageManifest) MarshalJSON() ([]byte, error) { + if err := im.assertValid(); err != nil { + return nil, err + } + return json.Marshal(imageManifest(im)) +} + +var imKindError = types.InvalidACKindError(ImageManifestKind) + +// assertValid performs extra assertions on an ImageManifest to ensure that +// fields are set appropriately, etc. It is used exclusively when marshalling +// and unmarshalling an ImageManifest. Most field-specific validation is +// performed through the individual types being marshalled; assertValid() +// should only deal with higher-level validation. +func (im *ImageManifest) assertValid() error { + if im.ACKind != ImageManifestKind { + return imKindError + } + if im.ACVersion.Empty() { + return errors.New(`acVersion must be set`) + } + if im.Name.Empty() { + return errors.New(`name must be set`) + } + return nil +} + +func (im *ImageManifest) GetLabel(name string) (val string, ok bool) { + return im.Labels.Get(name) +} + +func (im *ImageManifest) GetAnnotation(name string) (val string, ok bool) { + return im.Annotations.Get(name) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/image_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/image_test.go new file mode 100644 index 00000000000..fe9d676827c --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/image_test.go @@ -0,0 +1,48 @@ +package schema + +import "testing" + +func TestEmptyApp(t *testing.T) { + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.5.1", + "name": "example.com/test" + } + ` + + var im ImageManifest + + err := im.UnmarshalJSON([]byte(imj)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + // Marshal and Unmarshal to verify that no "app": {} is generated on + // Marshal and converted to empty struct on Unmarshal + buf, err := im.MarshalJSON() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = im.UnmarshalJSON(buf) + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestImageManifestMerge(t *testing.T) { + imj := `{"name": "example.com/test"}` + im := &ImageManifest{} + + if im.UnmarshalJSON([]byte(imj)) == nil { + t.Fatal("Manifest JSON without acKind and acVersion unmarshalled successfully") + } + + im = BlankImageManifest() + + err := im.UnmarshalJSON([]byte(imj)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/kind.go b/Godeps/_workspace/src/github.com/appc/spec/schema/kind.go new file mode 100644 index 00000000000..877d94d7e06 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/kind.go @@ -0,0 +1,28 @@ +package schema + +import ( + "encoding/json" + + "github.com/appc/spec/schema/types" +) + +type Kind struct { + ACVersion types.SemVer `json:"acVersion"` + ACKind types.ACKind `json:"acKind"` +} + +type kind Kind + +func (k *Kind) UnmarshalJSON(data []byte) error { + nk := kind{} + err := json.Unmarshal(data, &nk) + if err != nil { + return err + } + *k = Kind(nk) + return nil +} + +func (k Kind) MarshalJSON() ([]byte, error) { + return json.Marshal(kind(k)) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/pod.go b/Godeps/_workspace/src/github.com/appc/spec/schema/pod.go new file mode 100644 index 00000000000..ddf44c3db35 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/pod.go @@ -0,0 +1,146 @@ +package schema + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/appc/spec/schema/types" +) + +const PodManifestKind = types.ACKind("PodManifest") + +type PodManifest struct { + ACVersion types.SemVer `json:"acVersion"` + ACKind types.ACKind `json:"acKind"` + Apps AppList `json:"apps"` + Volumes []types.Volume `json:"volumes"` + Isolators []types.Isolator `json:"isolators"` + Annotations types.Annotations `json:"annotations"` + Ports []types.ExposedPort `json:"ports"` +} + +// podManifest is a model to facilitate extra validation during the +// unmarshalling of the PodManifest +type podManifest PodManifest + +func BlankPodManifest() *PodManifest { + return &PodManifest{ACKind: PodManifestKind, ACVersion: AppContainerVersion} +} + +func (pm *PodManifest) UnmarshalJSON(data []byte) error { + p := podManifest(*pm) + err := json.Unmarshal(data, &p) + if err != nil { + return err + } + npm := PodManifest(p) + if err := npm.assertValid(); err != nil { + return err + } + *pm = npm + return nil +} + +func (pm PodManifest) MarshalJSON() ([]byte, error) { + if err := pm.assertValid(); err != nil { + return nil, err + } + return json.Marshal(podManifest(pm)) +} + +var pmKindError = types.InvalidACKindError(PodManifestKind) + +// assertValid performs extra assertions on an PodManifest to +// ensure that fields are set appropriately, etc. It is used exclusively when +// marshalling and unmarshalling an PodManifest. Most +// field-specific validation is performed through the individual types being +// marshalled; assertValid() should only deal with higher-level validation. +func (pm *PodManifest) assertValid() error { + if pm.ACKind != PodManifestKind { + return pmKindError + } + return nil +} + +type AppList []RuntimeApp + +type appList AppList + +func (al *AppList) UnmarshalJSON(data []byte) error { + a := appList{} + err := json.Unmarshal(data, &a) + if err != nil { + return err + } + nal := AppList(a) + if err := nal.assertValid(); err != nil { + return err + } + *al = nal + return nil +} + +func (al AppList) MarshalJSON() ([]byte, error) { + if err := al.assertValid(); err != nil { + return nil, err + } + return json.Marshal(appList(al)) +} + +func (al AppList) assertValid() error { + seen := map[types.ACName]bool{} + for _, a := range al { + if _, ok := seen[a.Name]; ok { + return fmt.Errorf(`duplicate apps of name %q`, a.Name) + } + seen[a.Name] = true + } + return nil +} + +// Get retrieves an app by the specified name from the AppList; if there is +// no such app, nil is returned. The returned *RuntimeApp MUST be considered +// read-only. +func (al AppList) Get(name types.ACName) *RuntimeApp { + for _, a := range al { + if name.Equals(a.Name) { + aa := a + return &aa + } + } + return nil +} + +// Mount describes the mapping between a volume and an apps +// MountPoint that will be fulfilled at runtime. +type Mount struct { + Volume types.ACName `json:"volume"` + MountPoint types.ACName `json:"mountPoint"` +} + +func (r Mount) assertValid() error { + if r.Volume.Empty() { + return errors.New("volume must be set") + } + if r.MountPoint.Empty() { + return errors.New("mountPoint must be set") + } + return nil +} + +// RuntimeApp describes an application referenced in a PodManifest +type RuntimeApp struct { + Name types.ACName `json:"name"` + Image RuntimeImage `json:"image"` + App *types.App `json:"app,omitempty"` + Mounts []Mount `json:"mounts,omitempty"` + Annotations types.Annotations `json:"annotations,omitempty"` +} + +// RuntimeImage describes an image referenced in a RuntimeApp +type RuntimeImage struct { + Name *types.ACName `json:"name,omitempty"` + ID types.Hash `json:"id"` + Labels types.Labels `json:"labels,omitempty"` +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/pod_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/pod_test.go new file mode 100644 index 00000000000..46ac35fb2f4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/pod_test.go @@ -0,0 +1,59 @@ +package schema + +import ( + "testing" + + "github.com/appc/spec/schema/types" +) + +func TestPodManifestMerge(t *testing.T) { + pmj := `{}` + pm := &PodManifest{} + + if pm.UnmarshalJSON([]byte(pmj)) == nil { + t.Fatal("Manifest JSON without acKind and acVersion unmarshalled successfully") + } + + pm = BlankPodManifest() + + err := pm.UnmarshalJSON([]byte(pmj)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestAppList(t *testing.T) { + ri := RuntimeImage{ + ID: *types.NewHashSHA512([]byte{}), + } + al := AppList{ + RuntimeApp{ + Name: "foo", + Image: ri, + }, + RuntimeApp{ + Name: "bar", + Image: ri, + }, + } + if _, err := al.MarshalJSON(); err != nil { + t.Errorf("want err=nil, got %v", err) + } + dal := AppList{ + RuntimeApp{ + Name: "foo", + Image: ri, + }, + RuntimeApp{ + Name: "bar", + Image: ri, + }, + RuntimeApp{ + Name: "foo", + Image: ri, + }, + } + if _, err := dal.MarshalJSON(); err == nil { + t.Errorf("want err, got nil") + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/ackind.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/ackind.go new file mode 100644 index 00000000000..0be7c75500e --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/ackind.go @@ -0,0 +1,53 @@ +package types + +import ( + "encoding/json" + "fmt" +) + +var ( + ErrNoACKind = ACKindError("ACKind must be set") +) + +// ACKind wraps a string to define a field which must be set with one of +// several ACKind values. If it is unset, or has an invalid value, the field +// will refuse to marshal/unmarshal. +type ACKind string + +func (a ACKind) String() string { + return string(a) +} + +func (a ACKind) assertValid() error { + s := a.String() + switch s { + case "ImageManifest", "PodManifest": + return nil + case "": + return ErrNoACKind + default: + msg := fmt.Sprintf("bad ACKind: %s", s) + return ACKindError(msg) + } +} + +func (a ACKind) MarshalJSON() ([]byte, error) { + if err := a.assertValid(); err != nil { + return nil, err + } + return json.Marshal(a.String()) +} + +func (a *ACKind) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + na := ACKind(s) + if err := na.assertValid(); err != nil { + return err + } + *a = na + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/ackind_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/ackind_test.go new file mode 100644 index 00000000000..9adb4c4176d --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/ackind_test.go @@ -0,0 +1,79 @@ +package types + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestACKindMarshalBad(t *testing.T) { + tests := map[string]error{ + "Foo": ACKindError("bad ACKind: Foo"), + "ApplicationManifest": ACKindError("bad ACKind: ApplicationManifest"), + "": ErrNoACKind, + } + for in, werr := range tests { + a := ACKind(in) + b, gerr := json.Marshal(a) + if b != nil { + t.Errorf("ACKind(%q): want b=nil, got %v", in, b) + } + if jerr, ok := gerr.(*json.MarshalerError); !ok { + t.Errorf("expected JSONMarshalerError") + } else { + if e := jerr.Err; e != werr { + t.Errorf("err=%#v, want %#v", e, werr) + } + } + } +} + +func TestACKindMarshalGood(t *testing.T) { + for i, in := range []string{ + "ImageManifest", + "PodManifest", + } { + a := ACKind(in) + b, err := json.Marshal(a) + if !reflect.DeepEqual(b, []byte(`"`+in+`"`)) { + t.Errorf("#%d: marshalled=%v, want %v", i, b, []byte(in)) + } + if err != nil { + t.Errorf("#%d: err=%v, want nil", i, err) + } + } +} + +func TestACKindUnmarshalBad(t *testing.T) { + tests := []string{ + "ImageManifest", // Not a valid JSON-encoded string + `"garbage"`, + `"AppManifest"`, + `""`, + } + for i, in := range tests { + var a, b ACKind + err := a.UnmarshalJSON([]byte(in)) + if err == nil { + t.Errorf("#%d: err=nil, want non-nil", i) + } else if !reflect.DeepEqual(a, b) { + t.Errorf("#%d: a=%v, want empty", i, a) + } + } +} + +func TestACKindUnmarshalGood(t *testing.T) { + tests := map[string]ACKind{ + `"PodManifest"`: ACKind("PodManifest"), + `"ImageManifest"`: ACKind("ImageManifest"), + } + for in, w := range tests { + var a ACKind + err := json.Unmarshal([]byte(in), &a) + if err != nil { + t.Errorf("%v: err=%v, want nil", in, err) + } else if !reflect.DeepEqual(a, w) { + t.Errorf("%v: a=%v, want %v", in, a, w) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/acname.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/acname.go new file mode 100644 index 00000000000..3401e53c196 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/acname.go @@ -0,0 +1,131 @@ +package types + +import ( + "encoding/json" + "errors" + "regexp" + "strings" +) + +var ( + // ValidACName is a regular expression that defines a valid ACName + ValidACName = regexp.MustCompile("^[a-z0-9]+([-./][a-z0-9]+)*$") + + invalidChars = regexp.MustCompile("[^a-z0-9./-]") + invalidEdges = regexp.MustCompile("(^[./-]+)|([./-]+$)") + + ErrEmptyACName = ACNameError("ACName cannot be empty") + ErrInvalidEdge = ACNameError("ACName must start and end with only lower case " + + "alphanumeric characters") + ErrInvalidChar = ACNameError("ACName must contain only lower case " + + `alphanumeric characters plus ".", "-", "/"`) +) + +// ACName (an App-Container Name) is a format used by keys in different formats +// of the App Container Standard. An ACName is restricted to characters +// accepted by the DNS RFC[1] and "/"; all alphabetical characters must be +// lowercase only. Furthermore, the first and last character ("edges") must be +// alphanumeric, and an ACName cannot be empty. Programmatically, an ACName +// must conform to the regular expression ValidACName. +// +// [1] http://tools.ietf.org/html/rfc1123#page-13 +type ACName string + +func (n ACName) String() string { + return string(n) +} + +// Set sets the ACName to the given value, if it is valid; if not, +// an error is returned. +func (n *ACName) Set(s string) error { + nn, err := NewACName(s) + if err == nil { + *n = *nn + } + return err +} + +// Equals checks whether a given ACName is equal to this one. +func (n ACName) Equals(o ACName) bool { + return strings.ToLower(string(n)) == strings.ToLower(string(o)) +} + +// Empty returns a boolean indicating whether this ACName is empty. +func (n ACName) Empty() bool { + return n.String() == "" +} + +// NewACName generates a new ACName from a string. If the given string is +// not a valid ACName, nil and an error are returned. +func NewACName(s string) (*ACName, error) { + n := ACName(s) + if err := n.assertValid(); err != nil { + return nil, err + } + return &n, nil +} + +// MustACName generates a new ACName from a string, If the given string is +// not a valid ACName, it panics. +func MustACName(s string) *ACName { + n, err := NewACName(s) + if err != nil { + panic(err) + } + return n +} + +func (n ACName) assertValid() error { + s := string(n) + if len(s) == 0 { + return ErrEmptyACName + } + if invalidChars.MatchString(s) { + return ErrInvalidChar + } + if invalidEdges.MatchString(s) { + return ErrInvalidEdge + } + return nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface +func (n *ACName) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + nn, err := NewACName(s) + if err != nil { + return err + } + *n = *nn + return nil +} + +// MarshalJSON implements the json.Marshaler interface +func (n ACName) MarshalJSON() ([]byte, error) { + if err := n.assertValid(); err != nil { + return nil, err + } + return json.Marshal(n.String()) +} + +// SanitizeACName replaces every invalid ACName character in s with a dash +// making it a legal ACName string. If the character is an upper case letter it +// replaces it with its lower case. It also removes illegal edge characters +// (hyphens, periods and slashes). +// +// This is a helper function and its algorithm is not part of the spec. It +// should not be called without the user explicitly asking for a suggestion. +func SanitizeACName(s string) (string, error) { + s = strings.ToLower(s) + s = invalidChars.ReplaceAllString(s, "-") + s = invalidEdges.ReplaceAllString(s, "") + + if s == "" { + return "", errors.New("must contain at least one valid character") + } + + return s, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/acname_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/acname_test.go new file mode 100644 index 00000000000..7d118cfc261 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/acname_test.go @@ -0,0 +1,252 @@ +package types + +import ( + "encoding/json" + "reflect" + "testing" +) + +var ( + goodNames = []string{ + "asdf", + "foo-bar-baz", + "database", + "example.com/database", + "example.com/ourapp-1.0.0", + "sub-domain.example.com/org/product/release-1.0.0", + } + badNames = []string{ + "", + "foo#", + "EXAMPLE.com", + "foo.com/BAR", + "example.com/app_1", + "/app", + "app/", + "-app", + "app-", + ".app", + "app.", + } +) + +func TestNewACName(t *testing.T) { + for i, in := range goodNames { + l, err := NewACName(in) + if err != nil { + t.Errorf("#%d: got err=%v, want nil", i, err) + } + if l == nil { + t.Errorf("#%d: got l=nil, want non-nil", i) + } + } +} + +func TestNewACNameBad(t *testing.T) { + for i, in := range badNames { + l, err := NewACName(in) + if l != nil { + t.Errorf("#%d: got l=%v, want nil", i, l) + } + if err == nil { + t.Errorf("#%d: got err=nil, want non-nil", i) + } + } +} + +func TestMustACName(t *testing.T) { + for i, in := range goodNames { + l := MustACName(in) + if l == nil { + t.Errorf("#%d: got l=nil, want non-nil", i) + } + } +} + +func expectPanicMustACName(i int, in string, t *testing.T) { + defer func() { + recover() + }() + _ = MustACName(in) + t.Errorf("#%d: panic expected", i) +} + +func TestMustACNameBad(t *testing.T) { + for i, in := range badNames { + expectPanicMustACName(i, in, t) + } +} + +func TestSanitizeACName(t *testing.T) { + tests := map[string]string{ + "foo#": "foo", + "EXAMPLE.com": "example.com", + "foo.com/BAR": "foo.com/bar", + "example.com/app_1": "example.com/app-1", + "/app": "app", + "app/": "app", + "-app": "app", + "app-": "app", + ".app": "app", + "app.": "app", + "app///": "app", + "-/.app..": "app", + "-/app.name-test/-/": "app.name-test", + "sub-domain.example.com/org/product/release-1.0.0": "sub-domain.example.com/org/product/release-1.0.0", + } + for in, ex := range tests { + o, err := SanitizeACName(in) + if err != nil { + t.Errorf("got err=%v, want nil", err) + } + if o != ex { + t.Errorf("got l=%s, want %s", o, ex) + } + } +} + +func TestACNameSetGood(t *testing.T) { + tests := map[string]ACName{ + "blargh": ACName("blargh"), + "example.com/ourapp-1.0.0": ACName("example.com/ourapp-1.0.0"), + } + for in, w := range tests { + // Ensure an empty name is set appropriately + var a ACName + err := a.Set(in) + if err != nil { + t.Errorf("%v: got err=%v, want nil", in, err) + continue + } + if !reflect.DeepEqual(a, w) { + t.Errorf("%v: a=%v, want %v", in, a, w) + } + + // Ensure an existing name is overwritten + var b ACName = ACName("orig") + err = b.Set(in) + if err != nil { + t.Errorf("%v: got err=%v, want nil", in, err) + continue + } + if !reflect.DeepEqual(b, w) { + t.Errorf("%v: b=%v, want %v", in, b, w) + } + } +} + +func TestACNameSetBad(t *testing.T) { + for i, in := range badNames { + // Ensure an empty name stays empty + var a ACName + err := a.Set(in) + if err == nil { + t.Errorf("#%d: err=%v, want nil", i, err) + continue + } + if w := ACName(""); !reflect.DeepEqual(a, w) { + t.Errorf("%d: a=%v, want %v", i, a, w) + } + + // Ensure an existing name is not overwritten + var b ACName = ACName("orig") + err = b.Set(in) + if err == nil { + t.Errorf("#%d: err=%v, want nil", i, err) + continue + } + if w := ACName("orig"); !reflect.DeepEqual(b, w) { + t.Errorf("%d: b=%v, want %v", i, b, w) + } + } +} + +func TestSanitizeACNameBad(t *testing.T) { + tests := []string{ + "__", + "..", + "//", + "", + ".//-"} + for i, in := range tests { + l, err := SanitizeACName(in) + if l != "" { + t.Errorf("#%d: got l=%v, want nil", i, l) + } + if err == nil { + t.Errorf("#%d: got err=nil, want non-nil", i) + } + } +} + +func TestACNameUnmarshalBad(t *testing.T) { + tests := []string{ + "", + "garbage", + `""`, + `"EXAMPLE"`, + `"example.com/app_1"`, + } + for i, in := range tests { + var a, b ACName + err := a.UnmarshalJSON([]byte(in)) + if err == nil { + t.Errorf("#%d: err=nil, want non-nil", i) + } else if !reflect.DeepEqual(a, b) { + t.Errorf("#%d: a=%v, want empty", i, a) + } + } +} + +func TestACNameUnmarshalGood(t *testing.T) { + tests := map[string]ACName{ + `"example"`: ACName("example"), + `"foo.com/bar"`: ACName("foo.com/bar"), + } + for in, w := range tests { + var a ACName + err := json.Unmarshal([]byte(in), &a) + if err != nil { + t.Errorf("%v: err=%v, want nil", in, err) + } else if !reflect.DeepEqual(a, w) { + t.Errorf("%v: a=%v, want %v", in, a, w) + } + } +} + +func TestACNameMarshalBad(t *testing.T) { + tests := map[string]error{ + "Foo": ErrInvalidChar, + "foo#": ErrInvalidChar, + "/foo": ErrInvalidEdge, + "example.com/": ErrInvalidEdge, + "": ErrEmptyACName, + } + for in, werr := range tests { + a := ACName(in) + b, gerr := json.Marshal(a) + if b != nil { + t.Errorf("ACName(%q): want b=nil, got %v", in, b) + } + if jerr, ok := gerr.(*json.MarshalerError); !ok { + t.Errorf("expected JSONMarshalerError") + } else { + if e := jerr.Err; e != werr { + t.Errorf("err=%#v, want %#v", e, werr) + } + } + } +} + +func TestACNameMarshalGood(t *testing.T) { + for i, in := range goodNames { + a := ACName(in) + b, err := json.Marshal(a) + if !reflect.DeepEqual(b, []byte(`"`+in+`"`)) { + t.Errorf("#%d: marshalled=%v, want %v", i, b, []byte(in)) + } + if err != nil { + t.Errorf("#%d: err=%v, want nil", i, err) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/annotations.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/annotations.go new file mode 100644 index 00000000000..9c0d388c561 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/annotations.go @@ -0,0 +1,92 @@ +package types + +import ( + "encoding/json" + "fmt" +) + +type Annotations []Annotation + +type annotations Annotations + +type Annotation struct { + Name ACName `json:"name"` + Value string `json:"value"` +} + +func (a Annotations) assertValid() error { + seen := map[ACName]string{} + for _, anno := range a { + _, ok := seen[anno.Name] + if ok { + return fmt.Errorf(`duplicate annotations of name %q`, anno.Name) + } + seen[anno.Name] = anno.Value + } + if c, ok := seen["created"]; ok { + if _, err := NewDate(c); err != nil { + return err + } + } + if h, ok := seen["homepage"]; ok { + if _, err := NewURL(h); err != nil { + return err + } + } + if d, ok := seen["documentation"]; ok { + if _, err := NewURL(d); err != nil { + return err + } + } + + return nil +} + +func (a Annotations) MarshalJSON() ([]byte, error) { + if err := a.assertValid(); err != nil { + return nil, err + } + return json.Marshal(annotations(a)) +} + +func (a *Annotations) UnmarshalJSON(data []byte) error { + var ja annotations + if err := json.Unmarshal(data, &ja); err != nil { + return err + } + na := Annotations(ja) + if err := na.assertValid(); err != nil { + return err + } + *a = na + return nil +} + +// Retrieve the value of an annotation by the given name from Annotations, if +// it exists. +func (a Annotations) Get(name string) (val string, ok bool) { + for _, anno := range a { + if anno.Name.String() == name { + return anno.Value, true + } + } + return "", false +} + +// Set sets the value of an annotation by the given name, overwriting if one already exists. +func (a *Annotations) Set(name ACName, value string) { + for i, anno := range *a { + if anno.Name.Equals(name) { + (*a)[i] = Annotation{ + Name: name, + Value: value, + } + return + } + } + anno := Annotation{ + Name: name, + Value: value, + } + *a = append(*a, anno) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/annotations_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/annotations_test.go new file mode 100644 index 00000000000..af956dd2be4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/annotations_test.go @@ -0,0 +1,217 @@ +package types + +import ( + "reflect" + "testing" +) + +func makeAnno(n, v string) Annotation { + name, err := NewACName(n) + if err != nil { + panic(err) + } + return Annotation{ + Name: *name, + Value: v, + } +} + +func TestAnnotationsAssertValid(t *testing.T) { + tests := []struct { + in []Annotation + werr bool + }{ + // duplicate names should fail + { + []Annotation{ + makeAnno("foo", "bar"), + makeAnno("foo", "baz"), + }, + true, + }, + // bad created should fail + { + []Annotation{ + makeAnno("created", "garbage"), + }, + true, + }, + // bad homepage should fail + { + []Annotation{ + makeAnno("homepage", "not-A$@#URL"), + }, + true, + }, + // bad documentation should fail + { + []Annotation{ + makeAnno("documentation", "ftp://isnotallowed.com"), + }, + true, + }, + // good cases + { + []Annotation{ + makeAnno("created", "2004-05-14T23:11:14+00:00"), + makeAnno("documentation", "http://example.com/docs"), + }, + false, + }, + { + []Annotation{ + makeAnno("foo", "bar"), + makeAnno("homepage", "https://homepage.com"), + }, + false, + }, + // empty is OK + { + []Annotation{}, + false, + }, + } + for i, tt := range tests { + a := Annotations(tt.in) + err := a.assertValid() + if gerr := (err != nil); gerr != tt.werr { + t.Errorf("#%d: gerr=%t, want %t (err=%v)", i, gerr, tt.werr, err) + } + } +} + +func TestAnnotationsMarshal(t *testing.T) { + for i, tt := range []struct { + in []Annotation + wb []byte + werr bool + }{ + { + []Annotation{ + makeAnno("foo", "bar"), + makeAnno("foo", "baz"), + makeAnno("website", "http://example.com/anno"), + }, + nil, + true, + }, + { + []Annotation{ + makeAnno("a", "b"), + }, + []byte(`[{"name":"a","value":"b"}]`), + false, + }, + { + []Annotation{ + makeAnno("foo", "bar"), + makeAnno("website", "http://example.com/anno"), + }, + []byte(`[{"name":"foo","value":"bar"},{"name":"website","value":"http://example.com/anno"}]`), + false, + }, + } { + a := Annotations(tt.in) + b, err := a.MarshalJSON() + if !reflect.DeepEqual(b, tt.wb) { + t.Errorf("#%d: b=%s, want %s", i, b, tt.wb) + } + gerr := err != nil + if gerr != tt.werr { + t.Errorf("#%d: gerr=%t, want %t (err=%v)", i, gerr, tt.werr, err) + } + } +} + +func TestAnnotationsUnmarshal(t *testing.T) { + tests := []struct { + in string + wann *Annotations + werr bool + }{ + { + `garbage`, + &Annotations{}, + true, + }, + { + `[{"name":"a","value":"b"},{"name":"a","value":"b"}]`, + &Annotations{}, + true, + }, + { + `[{"name":"a","value":"b"}]`, + &Annotations{ + makeAnno("a", "b"), + }, + false, + }, + } + for i, tt := range tests { + a := &Annotations{} + err := a.UnmarshalJSON([]byte(tt.in)) + gerr := err != nil + if gerr != tt.werr { + t.Errorf("#%d: gerr=%t, want %t (err=%v)", i, gerr, tt.werr, err) + } + if !reflect.DeepEqual(a, tt.wann) { + t.Errorf("#%d: ann=%#v, want %#v", i, a, tt.wann) + } + } + +} + +func TestAnnotationsGet(t *testing.T) { + for i, tt := range []struct { + in string + wval string + wok bool + }{ + {"foo", "bar", true}, + {"website", "http://example.com/anno", true}, + {"baz", "", false}, + {"wuuf", "", false}, + } { + a := Annotations{ + makeAnno("foo", "bar"), + makeAnno("website", "http://example.com/anno"), + } + gval, gok := a.Get(tt.in) + if gval != tt.wval { + t.Errorf("#%d: val=%v, want %v", i, gval, tt.wval) + } + if gok != tt.wok { + t.Errorf("#%d: ok=%t, want %t", i, gok, tt.wok) + } + } +} + +func TestAnnotationsSet(t *testing.T) { + a := Annotations{} + + a.Set("foo", "bar") + w := Annotations{ + Annotation{ACName("foo"), "bar"}, + } + if !reflect.DeepEqual(w, a) { + t.Fatalf("want %v, got %v", w, a) + } + + a.Set("dog", "woof") + w = Annotations{ + Annotation{ACName("foo"), "bar"}, + Annotation{ACName("dog"), "woof"}, + } + if !reflect.DeepEqual(w, a) { + t.Fatalf("want %v, got %v", w, a) + } + + a.Set("foo", "baz") + w = Annotations{ + Annotation{ACName("foo"), "baz"}, + Annotation{ACName("dog"), "woof"}, + } + if !reflect.DeepEqual(w, a) { + t.Fatalf("want %v, got %v", w, a) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/app.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/app.go new file mode 100644 index 00000000000..e799322f841 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/app.go @@ -0,0 +1,75 @@ +package types + +import ( + "encoding/json" + "errors" + "fmt" + "path" +) + +type App struct { + Exec Exec `json:"exec"` + EventHandlers []EventHandler `json:"eventHandlers,omitempty"` + User string `json:"user"` + Group string `json:"group"` + WorkingDirectory string `json:"workingDirectory,omitempty"` + Environment Environment `json:"environment,omitempty"` + MountPoints []MountPoint `json:"mountPoints,omitempty"` + Ports []Port `json:"ports,omitempty"` + Isolators Isolators `json:"isolators,omitempty"` +} + +// app is a model to facilitate extra validation during the +// unmarshalling of the App +type app App + +func (a *App) UnmarshalJSON(data []byte) error { + ja := app(*a) + err := json.Unmarshal(data, &ja) + if err != nil { + return err + } + na := App(ja) + if err := na.assertValid(); err != nil { + return err + } + if na.Environment == nil { + na.Environment = make(Environment, 0) + } + *a = na + return nil +} + +func (a App) MarshalJSON() ([]byte, error) { + if err := a.assertValid(); err != nil { + return nil, err + } + return json.Marshal(app(a)) +} + +func (a *App) assertValid() error { + if err := a.Exec.assertValid(); err != nil { + return err + } + if a.User == "" { + return errors.New(`User is required`) + } + if a.Group == "" { + return errors.New(`Group is required`) + } + if !path.IsAbs(a.WorkingDirectory) && a.WorkingDirectory != "" { + return errors.New("WorkingDirectory must be an absolute path") + } + eh := make(map[string]bool) + for _, e := range a.EventHandlers { + name := e.Name + if eh[name] { + return fmt.Errorf("Only one eventHandler of name %q allowed", name) + } + eh[name] = true + } + if err := a.Environment.assertValid(); err != nil { + return err + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/app_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/app_test.go new file mode 100644 index 00000000000..fd1482dafbc --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/app_test.go @@ -0,0 +1,216 @@ +package types + +import ( + "reflect" + "testing" +) + +func TestAppValid(t *testing.T) { + tests := []App{ + App{ + Exec: []string{"/bin/httpd"}, + User: "0", + Group: "0", + WorkingDirectory: "/tmp", + }, + App{ + Exec: []string{"/app"}, + User: "0", + Group: "0", + EventHandlers: []EventHandler{ + {Name: "pre-start"}, + {Name: "post-stop"}, + }, + Environment: []EnvironmentVariable{ + {Name: "DEBUG", Value: "true"}, + }, + WorkingDirectory: "/tmp", + }, + App{ + Exec: []string{"/app", "arg1", "arg2"}, + User: "0", + Group: "0", + WorkingDirectory: "/tmp", + }, + } + for i, tt := range tests { + if err := tt.assertValid(); err != nil { + t.Errorf("#%d: err == %v, want nil", i, err) + } + } +} + +func TestAppExecInvalid(t *testing.T) { + tests := []App{ + App{ + Exec: nil, + }, + App{ + Exec: []string{}, + User: "0", + Group: "0", + }, + App{ + Exec: []string{"app"}, + User: "0", + Group: "0", + }, + App{ + Exec: []string{"bin/app", "arg1"}, + User: "0", + Group: "0", + }, + } + for i, tt := range tests { + if err := tt.assertValid(); err == nil { + t.Errorf("#%d: err == nil, want non-nil", i) + } + } +} + +func TestAppEventHandlersInvalid(t *testing.T) { + tests := []App{ + App{ + Exec: []string{"/bin/httpd"}, + User: "0", + Group: "0", + EventHandlers: []EventHandler{ + EventHandler{ + Name: "pre-start", + }, + EventHandler{ + Name: "pre-start", + }, + }, + }, + App{ + Exec: []string{"/bin/httpd"}, + User: "0", + Group: "0", + EventHandlers: []EventHandler{ + EventHandler{ + Name: "post-stop", + }, + EventHandler{ + Name: "pre-start", + }, + EventHandler{ + Name: "post-stop", + }, + }, + }, + } + for i, tt := range tests { + if err := tt.assertValid(); err == nil { + t.Errorf("#%d: err == nil, want non-nil", i) + } + } +} + +func TestUserGroupInvalid(t *testing.T) { + tests := []App{ + App{ + Exec: []string{"/app"}, + }, + App{ + Exec: []string{"/app"}, + User: "0", + }, + App{ + Exec: []string{"app"}, + Group: "0", + }, + } + for i, tt := range tests { + if err := tt.assertValid(); err == nil { + t.Errorf("#%d: err == nil, want non-nil", i) + } + } +} + +func TestAppWorkingDirectoryInvalid(t *testing.T) { + tests := []App{ + App{ + Exec: []string{"/app"}, + User: "foo", + Group: "bar", + WorkingDirectory: "stuff", + }, + App{ + Exec: []string{"/app"}, + User: "foo", + Group: "bar", + WorkingDirectory: "../home/fred", + }, + } + for i, tt := range tests { + if err := tt.assertValid(); err == nil { + t.Errorf("#%d: err == nil, want non-nil", i) + } + } +} + +func TestAppEnvironmentInvalid(t *testing.T) { + tests := []App{ + App{ + Exec: []string{"/app"}, + User: "foo", + Group: "bar", + Environment: Environment{ + EnvironmentVariable{"0DEBUG", "true"}, + }, + }, + } + for i, tt := range tests { + if err := tt.assertValid(); err == nil { + t.Errorf("#%d: err == nil, want non-nil", i) + } + } +} + +func TestAppUnmarshal(t *testing.T) { + tests := []struct { + in string + wann *App + werr bool + }{ + { + `garbage`, + &App{}, + true, + }, + { + `{"Exec":"not a list"}`, + &App{}, + true, + }, + { + `{"Exec":["notfullyqualified"]}`, + &App{}, + true, + }, + { + `{"Exec":["/a"],"User":"0","Group":"0"}`, + &App{ + Exec: Exec{ + "/a", + }, + User: "0", + Group: "0", + Environment: make(Environment, 0), + }, + false, + }, + } + for i, tt := range tests { + a := &App{} + err := a.UnmarshalJSON([]byte(tt.in)) + gerr := err != nil + if gerr != tt.werr { + t.Errorf("#%d: gerr=%t, want %t (err=%v)", i, gerr, tt.werr, err) + } + if !reflect.DeepEqual(a, tt.wann) { + t.Errorf("#%d: ann=%#v, want %#v", i, a, tt.wann) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/date.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/date.go new file mode 100644 index 00000000000..46ba27e06e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/date.go @@ -0,0 +1,46 @@ +package types + +import ( + "encoding/json" + "fmt" + "time" +) + +// Date wraps time.Time to marshal/unmarshal to/from JSON strings in strict +// accordance with RFC3339 +// TODO(jonboulle): golang's implementation seems slightly buggy here; +// according to http://tools.ietf.org/html/rfc3339#section-5.6 , applications +// may choose to separate the date and time with a space instead of a T +// character (for example, `date --rfc-3339` on GNU coreutils) - but this is +// considered an error by go's parser. File a bug? +type Date time.Time + +func NewDate(s string) (*Date, error) { + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return nil, fmt.Errorf("bad Date: %v", err) + } + d := Date(t) + return &d, nil +} + +func (d Date) String() string { + return time.Time(d).Format(time.RFC3339) +} + +func (d *Date) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + nd, err := NewDate(s) + if err != nil { + return err + } + *d = *nd + return nil +} + +func (d Date) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/date_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/date_test.go new file mode 100644 index 00000000000..4730874401d --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/date_test.go @@ -0,0 +1,66 @@ +package types + +import ( + "encoding/json" + "testing" + "time" +) + +var ( + pst = time.FixedZone("Pacific", -8*60*60) +) + +func TestUnmarshalDate(t *testing.T) { + tests := []struct { + in string + + wt time.Time + }{ + { + `"2004-05-14T23:11:14+00:00"`, + + time.Date(2004, 05, 14, 23, 11, 14, 0, time.UTC), + }, + { + `"2001-02-03T04:05:06Z"`, + + time.Date(2001, 02, 03, 04, 05, 06, 0, time.UTC), + }, + { + `"2014-11-14T17:36:54-08:00"`, + + time.Date(2014, 11, 14, 17, 36, 54, 0, pst), + }, + { + `"2004-05-14T23:11:14+00:00"`, + + time.Date(2004, 05, 14, 23, 11, 14, 0, time.UTC), + }, + } + for i, tt := range tests { + var d Date + if err := json.Unmarshal([]byte(tt.in), &d); err != nil { + t.Errorf("#%d: got err=%v, want nil", i, err) + } + if gt := time.Time(d); !gt.Equal(tt.wt) { + t.Errorf("#%d: got time=%v, want %v", i, gt, tt.wt) + } + } +} + +func TestUnmarshalDateBad(t *testing.T) { + tests := []string{ + `not a json string`, + `2014-11-14T17:36:54-08:00`, + `"garbage"`, + `"1416015188"`, + `"Fri Nov 14 17:53:02 PST 2014"`, + `"2014-11-1417:36:54"`, + } + for i, tt := range tests { + var d Date + if err := json.Unmarshal([]byte(tt), &d); err == nil { + t.Errorf("#%d: unexpected nil err", i) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/dependencies.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/dependencies.go new file mode 100644 index 00000000000..4f05ca890bd --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/dependencies.go @@ -0,0 +1,43 @@ +package types + +import ( + "encoding/json" + "errors" +) + +type Dependencies []Dependency + +type Dependency struct { + App ACName `json:"app"` + ImageID *Hash `json:"imageID,omitempty"` + Labels Labels `json:"labels,omitempty"` +} + +type dependency Dependency + +func (d Dependency) assertValid() error { + if len(d.App) < 1 { + return errors.New(`App cannot be empty`) + } + return nil +} + +func (d Dependency) MarshalJSON() ([]byte, error) { + if err := d.assertValid(); err != nil { + return nil, err + } + return json.Marshal(dependency(d)) +} + +func (d *Dependency) UnmarshalJSON(data []byte) error { + var jd dependency + if err := json.Unmarshal(data, &jd); err != nil { + return err + } + nd := Dependency(jd) + if err := nd.assertValid(); err != nil { + return err + } + *d = nd + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/dependencies_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/dependencies_test.go new file mode 100644 index 00000000000..9d39b2f7419 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/dependencies_test.go @@ -0,0 +1,26 @@ +package types + +import "testing" + +func TestEmptyHash(t *testing.T) { + dj := `{"app": "example.com/reduce-worker-base"}` + + var d Dependency + + err := d.UnmarshalJSON([]byte(dj)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Marshal to verify that marshalling works without validation errors + buf, err := d.MarshalJSON() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Unmarshal to verify that the generated json will not create wrong empty hash + err = d.UnmarshalJSON(buf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/doc.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/doc.go new file mode 100644 index 00000000000..d79a376cbc7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/doc.go @@ -0,0 +1,4 @@ +// Package types contains structs representing the various types in the app +// container specification. It is used by the [schema manifest types](../) +// to enforce validation. +package types diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/environment.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/environment.go new file mode 100644 index 00000000000..16c39f2c9c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/environment.go @@ -0,0 +1,96 @@ +package types + +import ( + "encoding/json" + "fmt" + "regexp" +) + +var ( + envPattern = regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*$") +) + +type Environment []EnvironmentVariable + +type environment Environment + +type EnvironmentVariable struct { + Name string `json:"name"` + Value string `json:"value"` +} + +func (ev EnvironmentVariable) assertValid() error { + if len(ev.Name) == 0 { + return fmt.Errorf(`environment variable name must not be empty`) + } + if !envPattern.MatchString(ev.Name) { + return fmt.Errorf(`environment variable does not have valid identifier %q`, ev.Name) + } + return nil +} + +func (e Environment) assertValid() error { + seen := map[string]bool{} + for _, env := range e { + if err := env.assertValid(); err != nil { + return err + } + _, ok := seen[env.Name] + if ok { + return fmt.Errorf(`duplicate environment variable of name %q`, env.Name) + } + seen[env.Name] = true + } + + return nil +} + +func (e Environment) MarshalJSON() ([]byte, error) { + if err := e.assertValid(); err != nil { + return nil, err + } + return json.Marshal(environment(e)) +} + +func (e *Environment) UnmarshalJSON(data []byte) error { + var je environment + if err := json.Unmarshal(data, &je); err != nil { + return err + } + ne := Environment(je) + if err := ne.assertValid(); err != nil { + return err + } + *e = ne + return nil +} + +// Retrieve the value of an environment variable by the given name from +// Environment, if it exists. +func (e Environment) Get(name string) (value string, ok bool) { + for _, env := range e { + if env.Name == name { + return env.Value, true + } + } + return "", false +} + +// Set sets the value of an environment variable by the given name, +// overwriting if one already exists. +func (e *Environment) Set(name string, value string) { + for i, env := range *e { + if env.Name == name { + (*e)[i] = EnvironmentVariable{ + Name: name, + Value: value, + } + return + } + } + env := EnvironmentVariable{ + Name: name, + Value: value, + } + *e = append(*e, env) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/environment_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/environment_test.go new file mode 100644 index 00000000000..54994a38e76 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/environment_test.go @@ -0,0 +1,62 @@ +package types + +import ( + "testing" +) + +func TestEnvironmentAssertValid(t *testing.T) { + tests := []struct { + env Environment + werr bool + }{ + // duplicate names should fail + { + Environment{ + EnvironmentVariable{"DEBUG", "true"}, + EnvironmentVariable{"DEBUG", "true"}, + }, + true, + }, + // empty name should fail + { + Environment{ + EnvironmentVariable{"", "value"}, + }, + true, + }, + // name beginning with digit should fail + { + Environment{ + EnvironmentVariable{"0DEBUG", "true"}, + }, + true, + }, + // name with non [A-Za-z0-9_] should fail + { + Environment{ + EnvironmentVariable{"VERBOSE-DEBUG", "true"}, + }, + true, + }, + // accepted environment variable forms + { + Environment{ + EnvironmentVariable{"DEBUG", "true"}, + }, + false, + }, + { + Environment{ + EnvironmentVariable{"_0_DEBUG_0_", "true"}, + }, + false, + }, + } + for i, test := range tests { + env := Environment(test.env) + err := env.assertValid() + if gerr := (err != nil); gerr != test.werr { + t.Errorf("#%d: gerr=%t, want %t (err=%v)", i, gerr, test.werr, err) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/errors.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/errors.go new file mode 100644 index 00000000000..6af44c159f2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/errors.go @@ -0,0 +1,28 @@ +package types + +import "fmt" + +// An ACKindError is returned when the wrong ACKind is set in a manifest +type ACKindError string + +func (e ACKindError) Error() string { + return string(e) +} + +func InvalidACKindError(kind ACKind) ACKindError { + return ACKindError(fmt.Sprintf("missing or bad ACKind (must be %#v)", kind)) +} + +// An ACVersionError is returned when a bad ACVersion is set in a manifest +type ACVersionError string + +func (e ACVersionError) Error() string { + return string(e) +} + +// An ACNameError is returned when a bad value is used for an ACName +type ACNameError string + +func (e ACNameError) Error() string { + return string(e) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/event_handler.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/event_handler.go new file mode 100644 index 00000000000..b0fef4f3808 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/event_handler.go @@ -0,0 +1,47 @@ +package types + +import ( + "encoding/json" + "errors" + "fmt" +) + +type EventHandler struct { + Name string `json:"name"` + Exec Exec `json:"exec"` +} + +type eventHandler EventHandler + +func (e EventHandler) assertValid() error { + s := e.Name + switch s { + case "pre-start", "post-stop": + return nil + case "": + return errors.New(`eventHandler "name" cannot be empty`) + default: + return fmt.Errorf(`bad eventHandler "name": %q`, s) + } +} + +func (e EventHandler) MarshalJSON() ([]byte, error) { + if err := e.assertValid(); err != nil { + return nil, err + } + return json.Marshal(eventHandler(e)) +} + +func (e *EventHandler) UnmarshalJSON(data []byte) error { + var je eventHandler + err := json.Unmarshal(data, &je) + if err != nil { + return err + } + ne := EventHandler(je) + if err := ne.assertValid(); err != nil { + return err + } + *e = ne + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/exec.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/exec.go new file mode 100644 index 00000000000..4a0cc092b42 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/exec.go @@ -0,0 +1,42 @@ +package types + +import ( + "encoding/json" + "errors" + "path/filepath" +) + +type Exec []string + +type exec Exec + +func (e Exec) assertValid() error { + if len(e) < 1 { + return errors.New(`Exec cannot be empty`) + } + if !filepath.IsAbs(e[0]) { + return errors.New(`Exec[0] must be absolute path`) + } + return nil +} + +func (e Exec) MarshalJSON() ([]byte, error) { + if err := e.assertValid(); err != nil { + return nil, err + } + return json.Marshal(exec(e)) +} + +func (e *Exec) UnmarshalJSON(data []byte) error { + var je exec + err := json.Unmarshal(data, &je) + if err != nil { + return err + } + ne := Exec(je) + if err := ne.assertValid(); err != nil { + return err + } + *e = ne + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/exec_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/exec_test.go new file mode 100644 index 00000000000..30007a13f09 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/exec_test.go @@ -0,0 +1,28 @@ +package types + +import "testing" + +func TestExecValid(t *testing.T) { + tests := []Exec{ + Exec{"/bin/httpd"}, + Exec{"/app"}, + Exec{"/app", "arg1", "arg2"}, + } + for i, tt := range tests { + if err := tt.assertValid(); err != nil { + t.Errorf("#%d: err == %v, want nil", i, err) + } + } +} + +func TestExecInvalid(t *testing.T) { + tests := []Exec{ + Exec{}, + Exec{"app"}, + } + for i, tt := range tests { + if err := tt.assertValid(); err == nil { + t.Errorf("#%d: err == nil, want non-nil", i) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/hash.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/hash.go new file mode 100644 index 00000000000..5c25d19bca6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/hash.go @@ -0,0 +1,104 @@ +package types + +import ( + "crypto/sha512" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" +) + +const ( + maxHashSize = (sha512.Size / 2) + len("sha512-") +) + +// Hash encodes a hash specified in a string of the form: +// "-" +// for example +// "sha512-06c733b1838136838e6d2d3e8fa5aea4c7905e92[...]" +// Valid types are currently: +// * sha512 +type Hash struct { + typ string + Val string +} + +func NewHash(s string) (*Hash, error) { + elems := strings.Split(s, "-") + if len(elems) != 2 { + return nil, errors.New("badly formatted hash string") + } + nh := Hash{ + typ: elems[0], + Val: elems[1], + } + if err := nh.assertValid(); err != nil { + return nil, err + } + return &nh, nil +} + +func (h Hash) String() string { + return fmt.Sprintf("%s-%s", h.typ, h.Val) +} + +func (h *Hash) Set(s string) error { + nh, err := NewHash(s) + if err == nil { + *h = *nh + } + return err +} + +func (h Hash) Empty() bool { + return reflect.DeepEqual(h, Hash{}) +} + +func (h Hash) assertValid() error { + switch h.typ { + case "sha512": + case "": + return fmt.Errorf("unexpected empty hash type") + default: + return fmt.Errorf("unrecognized hash type: %v", h.typ) + } + if h.Val == "" { + return fmt.Errorf("unexpected empty hash value") + } + return nil +} + +func (h *Hash) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + nh, err := NewHash(s) + if err != nil { + return err + } + *h = *nh + return nil +} + +func (h Hash) MarshalJSON() ([]byte, error) { + if err := h.assertValid(); err != nil { + return nil, err + } + return json.Marshal(h.String()) +} + +func NewHashSHA512(b []byte) *Hash { + h := sha512.New() + h.Write(b) + nh, _ := NewHash(fmt.Sprintf("sha512-%x", h.Sum(nil))) + return nh +} + +func ShortHash(hash string) string { + if len(hash) > maxHashSize { + return hash[:maxHashSize] + } + return hash +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/hash_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/hash_test.go new file mode 100644 index 00000000000..150b3df468b --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/hash_test.go @@ -0,0 +1,82 @@ +package types + +import ( + "encoding/json" + "testing" +) + +func TestMarshalHash(t *testing.T) { + tests := []struct { + typ string + val string + + wout string + }{ + { + "sha512", + "abcdefghi", + + `"sha512-abcdefghi"`, + }, + { + "sha512", + "06c733b1838136838e6d2d3e8fa5aea4c7905e92", + + `"sha512-06c733b1838136838e6d2d3e8fa5aea4c7905e92"`, + }, + } + for i, tt := range tests { + h := Hash{ + typ: tt.typ, + Val: tt.val, + } + b, err := json.Marshal(h) + if err != nil { + t.Errorf("#%d: unexpected err=%v", i, err) + } + if g := string(b); g != tt.wout { + t.Errorf("#%d: got string=%v, want %v", i, g, tt.wout) + } + } +} + +func TestMarshalHashBad(t *testing.T) { + tests := []struct { + typ string + val string + }{ + { + // empty value + "sha512", + "", + }, + { + // bad type + "sha1", + "abcdef", + }, + { + // empty type + "", + "abcdef", + }, + { + // empty empty + "", + "", + }, + } + for i, tt := range tests { + h := Hash{ + typ: tt.typ, + Val: tt.val, + } + g, err := json.Marshal(h) + if err == nil { + t.Errorf("#%d: unexpected nil err", i) + } + if g != nil { + t.Errorf("#%d: unexpected non-nil bytes: %v", i, g) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator.go new file mode 100644 index 00000000000..fefc8d125f6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator.go @@ -0,0 +1,89 @@ +package types + +import ( + "encoding/json" +) + +var ( + isolatorMap map[ACName]IsolatorValueConstructor +) + +func init() { + isolatorMap = make(map[ACName]IsolatorValueConstructor) +} + +type IsolatorValueConstructor func() IsolatorValue + +func AddIsolatorValueConstructor(n ACName, i IsolatorValueConstructor) { + isolatorMap[n] = i +} + +type Isolators []Isolator + +// GetByName returns the last isolator in the list by the given name. +func (is *Isolators) GetByName(name ACName) *Isolator { + var i Isolator + for j := len(*is) - 1; j >= 0; j-- { + i = []Isolator(*is)[j] + if i.Name == name { + return &i + } + } + return nil +} + +// Unrecognized returns a set of isolators that are not recognized. +// An isolator is not recognized if it has not had an associated +// constructor registered with AddIsolatorValueConstructor. +func (is *Isolators) Unrecognized() Isolators { + u := Isolators{} + for _, i := range *is { + if i.value == nil { + u = append(u, i) + } + } + return u +} + +type IsolatorValue interface { + UnmarshalJSON(b []byte) error + AssertValid() error +} +type Isolator struct { + Name ACName `json:"name"` + ValueRaw *json.RawMessage `json:"value"` + value IsolatorValue +} +type isolator Isolator + +func (i *Isolator) Value() IsolatorValue { + return i.value +} + +func (i *Isolator) UnmarshalJSON(b []byte) error { + var ii isolator + err := json.Unmarshal(b, &ii) + if err != nil { + return err + } + + var dst IsolatorValue + con, ok := isolatorMap[ii.Name] + if ok { + dst = con() + err = dst.UnmarshalJSON(*ii.ValueRaw) + if err != nil { + return err + } + err = dst.AssertValid() + if err != nil { + return err + } + } + + i.value = dst + i.ValueRaw = ii.ValueRaw + i.Name = ii.Name + + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_linux_specific.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_linux_specific.go new file mode 100644 index 00000000000..9e2e8dfa2cb --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_linux_specific.go @@ -0,0 +1,69 @@ +package types + +import ( + "encoding/json" + "errors" +) + +const ( + LinuxCapabilitiesRetainSetName = "os/linux/capabilities-retain-set" + LinuxCapabilitiesRevokeSetName = "os/linux/capabilities-revoke-set" +) + +func init() { + AddIsolatorValueConstructor(LinuxCapabilitiesRetainSetName, NewLinuxCapabilitiesRetainSet) + AddIsolatorValueConstructor(LinuxCapabilitiesRevokeSetName, NewLinuxCapabilitiesRevokeSet) +} + +type LinuxCapabilitiesSet interface { + Set() []LinuxCapability + AssertValid() error +} + +type LinuxCapability string +type linuxCapabilitiesSetValue struct { + Set []LinuxCapability `json:"set"` +} + +type linuxCapabilitiesSetBase struct { + val linuxCapabilitiesSetValue +} + +func (l linuxCapabilitiesSetBase) AssertValid() error { + if len(l.val.Set) == 0 { + return errors.New("set must be non-empty") + } + return nil +} + +func (l *linuxCapabilitiesSetBase) UnmarshalJSON(b []byte) error { + var v linuxCapabilitiesSetValue + err := json.Unmarshal(b, &v) + if err != nil { + return err + } + + l.val = v + + return err +} + +func (l linuxCapabilitiesSetBase) Set() []LinuxCapability { + return l.val.Set +} + +func NewLinuxCapabilitiesRetainSet() IsolatorValue { + return &LinuxCapabilitiesRetainSet{} +} + +type LinuxCapabilitiesRetainSet struct { + linuxCapabilitiesSetBase +} + +func NewLinuxCapabilitiesRevokeSet() IsolatorValue { + return &LinuxCapabilitiesRevokeSet{} +} + +type LinuxCapabilitiesRevokeSet struct { + linuxCapabilitiesSetBase +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_resources.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_resources.go new file mode 100644 index 00000000000..f930e441a97 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_resources.go @@ -0,0 +1,144 @@ +package types + +import ( + "encoding/json" + "errors" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource" +) + +var ( + ErrDefaultTrue = errors.New("default must be false") + ErrDefaultRequired = errors.New("default must be true") + ErrRequestNonEmpty = errors.New("request not supported by this resource, must be empty") +) + +const ( + ResourceBlockBandwidthName = "resource/block-bandwidth" + ResourceBlockIOPSName = "resource/block-iops" + ResourceCPUName = "resource/cpu" + ResourceMemoryName = "resource/memory" + ResourceNetworkBandwidthName = "resource/network-bandwidth" +) + +func init() { + AddIsolatorValueConstructor(ResourceBlockBandwidthName, NewResourceBlockBandwidth) + AddIsolatorValueConstructor(ResourceBlockIOPSName, NewResourceBlockIOPS) + AddIsolatorValueConstructor(ResourceCPUName, NewResourceCPU) + AddIsolatorValueConstructor(ResourceMemoryName, NewResourceMemory) + AddIsolatorValueConstructor(ResourceNetworkBandwidthName, NewResourceNetworkBandwidth) +} + +func NewResourceBlockBandwidth() IsolatorValue { + return &ResourceBlockBandwidth{} +} +func NewResourceBlockIOPS() IsolatorValue { + return &ResourceBlockIOPS{} +} +func NewResourceCPU() IsolatorValue { + return &ResourceCPU{} +} +func NewResourceNetworkBandwidth() IsolatorValue { + return &ResourceNetworkBandwidth{} +} +func NewResourceMemory() IsolatorValue { + return &ResourceMemory{} +} + +type Resource interface { + Limit() *resource.Quantity + Request() *resource.Quantity + Default() bool +} + +type ResourceBase struct { + val resourceValue +} + +type resourceValue struct { + Default bool `json:"default"` + Request *resource.Quantity `json:"request"` + Limit *resource.Quantity `json:"limit"` +} + +func (r ResourceBase) Limit() *resource.Quantity { + return r.val.Limit +} +func (r ResourceBase) Request() *resource.Quantity { + return r.val.Request +} +func (r ResourceBase) Default() bool { + return r.val.Default +} + +func (r *ResourceBase) UnmarshalJSON(b []byte) error { + return json.Unmarshal(b, &r.val) +} + +func (r ResourceBase) AssertValid() error { + return nil +} + +type ResourceBlockBandwidth struct { + ResourceBase +} + +func (r ResourceBlockBandwidth) AssertValid() error { + if r.Default() != true { + return ErrDefaultRequired + } + if r.Request() != nil { + return ErrRequestNonEmpty + } + return nil +} + +type ResourceBlockIOPS struct { + ResourceBase +} + +func (r ResourceBlockIOPS) AssertValid() error { + if r.Default() != true { + return ErrDefaultRequired + } + if r.Request() != nil { + return ErrRequestNonEmpty + } + return nil +} + +type ResourceCPU struct { + ResourceBase +} + +func (r ResourceCPU) AssertValid() error { + if r.Default() != false { + return ErrDefaultTrue + } + return nil +} + +type ResourceMemory struct { + ResourceBase +} + +func (r ResourceMemory) AssertValid() error { + if r.Default() != false { + return ErrDefaultTrue + } + return nil +} + +type ResourceNetworkBandwidth struct { + ResourceBase +} + +func (r ResourceNetworkBandwidth) AssertValid() error { + if r.Default() != true { + return ErrDefaultRequired + } + if r.Request() != nil { + return ErrRequestNonEmpty + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_test.go new file mode 100644 index 00000000000..087a7db4139 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/isolator_test.go @@ -0,0 +1,236 @@ +package types + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestIsolatorUnmarshal(t *testing.T) { + tests := []struct { + msg string + werr bool + }{ + { + `{ + "name": "os/linux/capabilities-retain-set", + "value": {"set": ["CAP_KILL"]} + }`, + false, + }, + { + `{ + "name": "os/linux/capabilities-retain-set", + "value": {"set": ["CAP_PONIES"]} + }`, + false, + }, + { + `{ + "name": "os/linux/capabilities-retain-set", + "value": {"set": []} + }`, + true, + }, + { + `{ + "name": "os/linux/capabilities-retain-set", + "value": {"set": "CAP_PONIES"} + }`, + true, + }, + { + `{ + "name": "resource/block-bandwidth", + "value": {"default": true, "limit": "1G"} + }`, + false, + }, + { + `{ + "name": "resource/block-bandwidth", + "value": {"default": false, "limit": "1G"} + }`, + true, + }, + { + `{ + "name": "resource/block-bandwidth", + "value": {"request": "30G", "limit": "1G"} + }`, + true, + }, + { + `{ + "name": "resource/block-iops", + "value": {"default": true, "limit": "1G"} + }`, + false, + }, + { + `{ + "name": "resource/block-iops", + "value": {"default": false, "limit": "1G"} + }`, + true, + }, + { + `{ + "name": "resource/block-iops", + "value": {"request": "30G", "limit": "1G"} + }`, + true, + }, + { + `{ + "name": "resource/cpu", + "value": {"request": "30", "limit": "1"} + }`, + false, + }, + { + `{ + "name": "resource/memory", + "value": {"request": "1G", "limit": "2Gi"} + }`, + false, + }, + { + `{ + "name": "resource/memory", + "value": {"default": true, "request": "1G", "limit": "2G"} + }`, + true, + }, + { + `{ + "name": "resource/network-bandwidth", + "value": {"default": true, "limit": "1G"} + }`, + false, + }, + { + `{ + "name": "resource/network-bandwidth", + "value": {"default": false, "limit": "1G"} + }`, + true, + }, + { + `{ + "name": "resource/network-bandwidth", + "value": {"request": "30G", "limit": "1G"} + }`, + true, + }, + } + + for i, tt := range tests { + var ii Isolator + err := ii.UnmarshalJSON([]byte(tt.msg)) + if gerr := (err != nil); gerr != tt.werr { + t.Errorf("#%d: gerr=%t, want %t (err=%v)", i, gerr, tt.werr, err) + } + } +} + +func TestIsolatorsGetByName(t *testing.T) { + ex := ` + [ + { + "name": "resource/cpu", + "value": {"request": "30", "limit": "1"} + }, + { + "name": "resource/memory", + "value": {"request": "1G", "limit": "2Gi"} + }, + { + "name": "os/linux/capabilities-retain-set", + "value": {"set": ["CAP_KILL"]} + }, + { + "name": "os/linux/capabilities-revoke-set", + "value": {"set": ["CAP_KILL"]} + } + ] + ` + + tests := []struct { + name ACName + wlimit int64 + wrequest int64 + wset []LinuxCapability + }{ + {"resource/cpu", 1, 30, nil}, + {"resource/memory", 2147483648, 1000000000, nil}, + {"os/linux/capabilities-retain-set", 0, 0, []LinuxCapability{"CAP_KILL"}}, + {"os/linux/capabilities-revoke-set", 0, 0, []LinuxCapability{"CAP_KILL"}}, + } + + var is Isolators + err := json.Unmarshal([]byte(ex), &is) + if err != nil { + panic(err) + } + + if len(is) < 2 { + t.Fatalf("too few items %v", len(is)) + } + + for i, tt := range tests { + c := is.GetByName(tt.name) + if c == nil { + t.Fatalf("can't find item %v in %v items", tt.name, len(is)) + } + switch v := c.Value().(type) { + case Resource: + var r Resource = v + glimit := r.Limit() + grequest := r.Request() + if glimit.Value() != tt.wlimit || grequest.Value() != tt.wrequest { + t.Errorf("#%d: glimit=%v, want %v, grequest=%v, want %v", i, glimit.Value(), tt.wlimit, grequest.Value(), tt.wrequest) + } + case LinuxCapabilitiesSet: + var s LinuxCapabilitiesSet = v + if !reflect.DeepEqual(s.Set(), tt.wset) { + t.Errorf("#%d: gset=%v, want %v", i, s.Set(), tt.wset) + } + + default: + panic("unexecpected type") + } + } +} + +func TestIsolatorUnrecognized(t *testing.T) { + msg := ` + [{ + "name": "resource/network-bandwidth", + "value": {"default": true, "limit": "1G"} + }, + { + "name": "resource/network-ponies", + "value": 0 + }]` + + ex := Isolators{ + {Name: "resource/network-ponies"}, + } + + is := Isolators{} + if err := json.Unmarshal([]byte(msg), &is); err != nil { + t.Fatalf("failed to unmarshal isolators: %v", err) + } + + u := is.Unrecognized() + if len(u) != len(ex) { + t.Errorf("unrecognized isolator list is wrong len: want %v, got %v", len(ex), len(u)) + } + + for i, e := range ex { + if e.Name != u[i].Name { + t.Errorf("unrecognized isolator list mismatch: want %v, got %v", e.Name, u[i].Name) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/labels.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/labels.go new file mode 100644 index 00000000000..45f4c8eda98 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/labels.go @@ -0,0 +1,114 @@ +package types + +import ( + "encoding/json" + "fmt" + "sort" +) + +var ValidOSArch = map[string][]string{ + "linux": {"amd64", "i386", "aarch64", "armv7l", "armv7b"}, + "freebsd": {"amd64", "i386", "arm"}, + "darwin": {"x86_64", "i386"}, +} + +type Labels []Label + +type labels Labels + +type Label struct { + Name ACName `json:"name"` + Value string `json:"value"` +} + +func (l Labels) assertValid() error { + seen := map[ACName]string{} + for _, lbl := range l { + if lbl.Name == "name" { + return fmt.Errorf(`invalid label name: "name"`) + } + _, ok := seen[lbl.Name] + if ok { + return fmt.Errorf(`duplicate labels of name %q`, lbl.Name) + } + seen[lbl.Name] = lbl.Value + } + if os, ok := seen["os"]; ok { + if validArchs, ok := ValidOSArch[os]; !ok { + // Not a whitelisted OS. TODO: how to warn rather than fail? + validOses := make([]string, 0, len(ValidOSArch)) + for validOs := range ValidOSArch { + validOses = append(validOses, validOs) + } + sort.Strings(validOses) + return fmt.Errorf(`bad os %#v (must be one of: %v)`, os, validOses) + } else { + // Whitelisted OS. We check arch here, as arch makes sense only + // when os is defined. + if arch, ok := seen["arch"]; ok { + found := false + for _, validArch := range validArchs { + if arch == validArch { + found = true + break + } + } + if !found { + return fmt.Errorf(`bad arch %#v for %v (must be one of: %v)`, arch, os, validArchs) + } + } + } + } + return nil +} + +func (l Labels) MarshalJSON() ([]byte, error) { + if err := l.assertValid(); err != nil { + return nil, err + } + return json.Marshal(labels(l)) +} + +func (l *Labels) UnmarshalJSON(data []byte) error { + var jl labels + if err := json.Unmarshal(data, &jl); err != nil { + return err + } + nl := Labels(jl) + if err := nl.assertValid(); err != nil { + return err + } + *l = nl + return nil +} + +// Get retrieves the value of the label by the given name from Labels, if it exists +func (l Labels) Get(name string) (val string, ok bool) { + for _, lbl := range l { + if lbl.Name.String() == name { + return lbl.Value, true + } + } + return "", false +} + +// ToMap creates a map[ACName]string. +func (l Labels) ToMap() map[ACName]string { + labelsMap := make(map[ACName]string) + for _, lbl := range l { + labelsMap[lbl.Name] = lbl.Value + } + return labelsMap +} + +// LabelsFromMap creates Labels from a map[ACName]string +func LabelsFromMap(labelsMap map[ACName]string) (Labels, error) { + labels := Labels{} + for n, v := range labelsMap { + labels = append(labels, Label{Name: n, Value: v}) + } + if err := labels.assertValid(); err != nil { + return nil, err + } + return labels, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/labels_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/labels_test.go new file mode 100644 index 00000000000..e175275e14a --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/labels_test.go @@ -0,0 +1,78 @@ +package types + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestLabels(t *testing.T) { + tests := []struct { + in string + errPrefix string + }{ + { + `[{"name": "os", "value": "linux"}, {"name": "arch", "value": "amd64"}]`, + "", + }, + { + `[{"name": "os", "value": "linux"}, {"name": "arch", "value": "aarch64"}]`, + "", + }, + { + `[{"name": "os", "value": "linux"}, {"name": "arch", "value": "armv7l"}]`, + "", + }, + { + `[{"name": "os", "value": "linux"}, {"name": "arch", "value": "armv7b"}]`, + "", + }, + { + `[{"name": "os", "value": "freebsd"}, {"name": "arch", "value": "amd64"}]`, + "", + }, + { + `[{"name": "os", "value": "OS/360"}, {"name": "arch", "value": "S/360"}]`, + `bad os "OS/360"`, + }, + { + `[{"name": "os", "value": "freebsd"}, {"name": "arch", "value": "armv7b"}]`, + `bad arch "armv7b" for freebsd`, + }, + { + `[{"name": "os", "value": "linux"}, {"name": "arch", "value": "arm"}]`, + `bad arch "arm" for linux`, + }, + { + `[{"name": "name"}]`, + `invalid label name: "name"`, + }, + { + `[{"name": "os", "value": "linux"}, {"name": "os", "value": "freebsd"}]`, + `duplicate labels of name "os"`, + }, + { + `[{"name": "arch", "value": "amd64"}, {"name": "os", "value": "freebsd"}, {"name": "arch", "value": "x86_64"}]`, + `duplicate labels of name "arch"`, + }, + { + `[]`, + "", + }, + } + for i, tt := range tests { + var l Labels + if err := json.Unmarshal([]byte(tt.in), &l); err != nil { + if tt.errPrefix == "" { + t.Errorf("#%d: got err=%v, expected no error", i, err) + } else if !strings.HasPrefix(err.Error(), tt.errPrefix) { + t.Errorf("#%d: got err=%v, expected prefix %#v", i, err, tt.errPrefix) + } + } else { + t.Log(l) + if tt.errPrefix != "" { + t.Errorf("#%d: got no err, expected prefix %#v", i, tt.errPrefix) + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/mountpoint.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/mountpoint.go new file mode 100644 index 00000000000..c405292a454 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/mountpoint.go @@ -0,0 +1,72 @@ +package types + +import ( + "errors" + "fmt" + "net/url" + "strconv" + "strings" +) + +type MountPoint struct { + Name ACName `json:"name"` + Path string `json:"path"` + ReadOnly bool `json:"readOnly,omitempty"` +} + +func (mount MountPoint) assertValid() error { + if mount.Name.Empty() { + return errors.New("name must be set") + } + if len(mount.Path) == 0 { + return errors.New("path must be set") + } + return nil +} + +// MountPointFromString takes a command line mountpoint parameter and returns a mountpoint +// +// It is useful for actool patch-manifest --mounts +// +// Example mountpoint parameters: +// database,path=/tmp,readOnly=true +func MountPointFromString(mp string) (*MountPoint, error) { + var mount MountPoint + + mp = "name=" + mp + v, err := url.ParseQuery(strings.Replace(mp, ",", "&", -1)) + if err != nil { + return nil, err + } + for key, val := range v { + if len(val) > 1 { + return nil, fmt.Errorf("label %s with multiple values %q", key, val) + } + + // TOOD(philips): make this less hardcoded + switch key { + case "name": + acn, err := NewACName(val[0]) + if err != nil { + return nil, err + } + mount.Name = *acn + case "path": + mount.Path = val[0] + case "readOnly": + ro, err := strconv.ParseBool(val[0]) + if err != nil { + return nil, err + } + mount.ReadOnly = ro + default: + return nil, fmt.Errorf("unknown mountpoint parameter %q", key) + } + } + err = mount.assertValid() + if err != nil { + return nil, err + } + + return &mount, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/mountpoint_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/mountpoint_test.go new file mode 100644 index 00000000000..c40a6449a68 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/mountpoint_test.go @@ -0,0 +1,69 @@ +package types + +import ( + "reflect" + "testing" +) + +func TestMountPointFromString(t *testing.T) { + tests := []struct { + s string + mount MountPoint + }{ + { + "foobar,path=/tmp", + MountPoint{ + Name: "foobar", + Path: "/tmp", + ReadOnly: false, + }, + }, + { + "foobar,path=/tmp,readOnly=false", + MountPoint{ + Name: "foobar", + Path: "/tmp", + ReadOnly: false, + }, + }, + { + "foobar,path=/tmp,readOnly=true", + MountPoint{ + Name: "foobar", + Path: "/tmp", + ReadOnly: true, + }, + }, + } + for i, tt := range tests { + mount, err := MountPointFromString(tt.s) + if err != nil { + t.Errorf("#%d: got err=%v, want nil", i, err) + } + if !reflect.DeepEqual(*mount, tt.mount) { + t.Errorf("#%d: mount=%v, want %v", i, *mount, tt.mount) + } + } +} + +func TestMountPointFromStringBad(t *testing.T) { + tests := []string{ + "#foobar,path=/tmp", + "foobar,path=/tmp,readOnly=true,asdf=asdf", + "foobar,path=/tmp,readOnly=maybe", + "foobar,path=/tmp,readOnly=", + "foobar,path=", + "foobar", + "", + ",path=/", + } + for i, in := range tests { + l, err := MountPointFromString(in) + if l != nil { + t.Errorf("#%d: got l=%v, want nil", i, l) + } + if err == nil { + t.Errorf("#%d: got err=nil, want non-nil", i) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/port.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/port.go new file mode 100644 index 00000000000..bfc06f8a0ab --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/port.go @@ -0,0 +1,13 @@ +package types + +type Port struct { + Name ACName `json:"name"` + Protocol string `json:"protocol"` + Port uint `json:"port"` + SocketActivated bool `json:"socketActivated"` +} + +type ExposedPort struct { + Name ACName `json:"name"` + HostPort uint `json:"hostPort"` +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/semver.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/semver.go new file mode 100644 index 00000000000..2c359688d63 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/semver.go @@ -0,0 +1,62 @@ +package types + +import ( + "encoding/json" + + "github.com/coreos/go-semver/semver" +) + +var ( + ErrNoZeroSemVer = ACVersionError("SemVer cannot be zero") + ErrBadSemVer = ACVersionError("SemVer is bad") +) + +// SemVer implements the Unmarshaler interface to define a field that must be +// a semantic version string +// TODO(jonboulle): extend upstream instead of wrapping? +type SemVer semver.Version + +// NewSemVer generates a new SemVer from a string. If the given string does +// not represent a valid SemVer, nil and an error are returned. +func NewSemVer(s string) (*SemVer, error) { + nsv, err := semver.NewVersion(s) + if err != nil { + return nil, ErrBadSemVer + } + v := SemVer(*nsv) + if v.Empty() { + return nil, ErrNoZeroSemVer + } + return &v, nil +} + +func (sv SemVer) String() string { + s := semver.Version(sv) + return s.String() +} + +func (sv SemVer) Empty() bool { + return semver.Version(sv) == semver.Version{} +} + +// UnmarshalJSON implements the json.Unmarshaler interface +func (sv *SemVer) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + v, err := NewSemVer(s) + if err != nil { + return err + } + *sv = *v + return nil +} + +// MarshalJSON implements the json.Marshaler interface +func (sv SemVer) MarshalJSON() ([]byte, error) { + if sv.Empty() { + return nil, ErrNoZeroSemVer + } + return json.Marshal(sv.String()) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/semver_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/semver_test.go new file mode 100644 index 00000000000..190c3d1cfcc --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/semver_test.go @@ -0,0 +1,115 @@ +package types + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/coreos/go-semver/semver" +) + +func TestMarshalSemver(t *testing.T) { + tests := []struct { + sv SemVer + + wd []byte + }{ + { + SemVer(semver.Version{Major: 1}), + + []byte(`"1.0.0"`), + }, + { + SemVer(semver.Version{Major: 3, Minor: 2, Patch: 1}), + + []byte(`"3.2.1"`), + }, + { + SemVer(semver.Version{Major: 3, Minor: 2, Patch: 1, PreRelease: "foo"}), + + []byte(`"3.2.1-foo"`), + }, + { + SemVer(semver.Version{Major: 1, Minor: 2, Patch: 3, PreRelease: "alpha", Metadata: "git"}), + + []byte(`"1.2.3-alpha+git"`), + }, + } + for i, tt := range tests { + d, err := json.Marshal(tt.sv) + if !reflect.DeepEqual(d, tt.wd) { + t.Errorf("#%d: d=%v, want %v", i, string(d), string(tt.wd)) + } + if err != nil { + t.Errorf("#%d: err=%v, want nil", i, err) + } + } +} + +func TestUnmarshalSemver(t *testing.T) { + tests := []struct { + d []byte + + wsv SemVer + werr bool + }{ + { + []byte(`"1.0.0"`), + + SemVer(semver.Version{Major: 1}), + false, + }, + { + []byte(`"3.2.1"`), + SemVer(semver.Version{Major: 3, Minor: 2, Patch: 1}), + + false, + }, + { + []byte(`"3.2.1-foo"`), + + SemVer(semver.Version{Major: 3, Minor: 2, Patch: 1, PreRelease: "foo"}), + false, + }, + { + []byte(`"1.2.3-alpha+git"`), + + SemVer(semver.Version{Major: 1, Minor: 2, Patch: 3, PreRelease: "alpha", Metadata: "git"}), + false, + }, + { + []byte(`"1"`), + + SemVer{}, + true, + }, + { + []byte(`"1.2.3.4"`), + + SemVer{}, + true, + }, + { + []byte(`1.2.3`), + + SemVer{}, + true, + }, + { + []byte(`"v1.2.3"`), + + SemVer{}, + true, + }, + } + for i, tt := range tests { + var sv SemVer + err := json.Unmarshal(tt.d, &sv) + if !reflect.DeepEqual(sv, tt.wsv) { + t.Errorf("#%d: semver=%#v, want %#v", i, sv, tt.wsv) + } + if gerr := (err != nil); gerr != tt.werr { + t.Errorf("#%d: err==%v, want errstate %t", i, err, tt.werr) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/url.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/url.go new file mode 100644 index 00000000000..391c295d47d --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/url.go @@ -0,0 +1,57 @@ +package types + +import ( + "encoding/json" + "fmt" + "net/url" +) + +// URL wraps url.URL to marshal/unmarshal to/from JSON strings and enforce +// that the scheme is HTTP/HTTPS only +type URL url.URL + +func NewURL(s string) (*URL, error) { + uu, err := url.Parse(s) + if err != nil { + return nil, fmt.Errorf("bad URL: %v", err) + } + nu := URL(*uu) + if err := nu.assertValidScheme(); err != nil { + return nil, err + } + return &nu, nil +} + +func (u URL) String() string { + uu := url.URL(u) + return uu.String() +} + +func (u URL) assertValidScheme() error { + switch u.Scheme { + case "http", "https": + return nil + default: + return fmt.Errorf("bad URL scheme, must be http/https") + } +} + +func (u *URL) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + nu, err := NewURL(s) + if err != nil { + return err + } + *u = *nu + return nil +} + +func (u URL) MarshalJSON() ([]byte, error) { + if err := u.assertValidScheme(); err != nil { + return nil, err + } + return json.Marshal(u.String()) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/url_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/url_test.go new file mode 100644 index 00000000000..52274967a14 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/url_test.go @@ -0,0 +1,123 @@ +package types + +import ( + "encoding/json" + "net/url" + "reflect" + "testing" +) + +func mustParseURL(t *testing.T, s string) url.URL { + u, err := url.Parse(s) + if err != nil { + t.Fatalf("error parsing URL: %v", err) + } + return *u +} + +func TestMarshalURL(t *testing.T) { + tests := []struct { + u url.URL + + w string + }{ + { + mustParseURL(t, "http://foo.com"), + + `"http://foo.com"`, + }, + { + mustParseURL(t, "http://foo.com/huh/what?is=this"), + + `"http://foo.com/huh/what?is=this"`, + }, + { + mustParseURL(t, "https://example.com/bar"), + + `"https://example.com/bar"`, + }, + } + for i, tt := range tests { + u := URL(tt.u) + b, err := json.Marshal(u) + if g := string(b); g != tt.w { + t.Errorf("#%d: got %q, want %q", i, g, tt.w) + } + if err != nil { + t.Errorf("#%d: err=%v, want nil", i, err) + } + + } +} + +func TestMarshalURLBad(t *testing.T) { + tests := []url.URL{ + mustParseURL(t, "ftp://foo.com"), + mustParseURL(t, "unix:///hello"), + } + for i, tt := range tests { + u := URL(tt) + b, err := json.Marshal(u) + if b != nil { + t.Errorf("#%d: got %v, want nil", i, b) + } + if err == nil { + t.Errorf("#%d: got unexpected err=nil", i) + } + } +} + +func TestUnmarshalURL(t *testing.T) { + tests := []struct { + in string + + w URL + }{ + { + `"http://foo.com"`, + + URL(mustParseURL(t, "http://foo.com")), + }, + { + `"http://yis.com/hello?goodbye=yes"`, + + URL(mustParseURL(t, "http://yis.com/hello?goodbye=yes")), + }, + { + `"https://ohai.net"`, + + URL(mustParseURL(t, "https://ohai.net")), + }, + } + for i, tt := range tests { + var g URL + err := json.Unmarshal([]byte(tt.in), &g) + if err != nil { + t.Errorf("#%d: want err=nil, got %v", i, err) + } + if !reflect.DeepEqual(g, tt.w) { + t.Errorf("#%d: got url=%v, want %v", i, g, tt.w) + } + } +} + +func TestUnmarshalURLBad(t *testing.T) { + var empty = URL{} + tests := []string{ + "badjson", + "http://google.com", + `"ftp://example.com"`, + `"unix://file.net"`, + `"not a url"`, + } + for i, tt := range tests { + var g URL + err := json.Unmarshal([]byte(tt), &g) + if err == nil { + t.Errorf("#%d: want err, got nil", i) + } + if !reflect.DeepEqual(g, empty) { + t.Errorf("#%d: got %v, want %v", i, g, empty) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/uuid.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/uuid.go new file mode 100644 index 00000000000..ec9f7996a4d --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/uuid.go @@ -0,0 +1,78 @@ +package types + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" +) + +var ( + ErrNoEmptyUUID = errors.New("UUID cannot be empty") +) + +// UUID encodes an RFC4122-compliant UUID, marshaled to/from a string +// TODO(jonboulle): vendor a package for this? +// TODO(jonboulle): consider more flexibility in input string formats. +// Right now, we only accept: +// "6733C088-A507-4694-AABF-EDBE4FC5266F" +// "6733C088A5074694AABFEDBE4FC5266F" +type UUID [16]byte + +func (u UUID) String() string { + return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:16]) +} + +func (u *UUID) Set(s string) error { + nu, err := NewUUID(s) + if err == nil { + *u = *nu + } + return err +} + +// NewUUID generates a new UUID from the given string. If the string does not +// represent a valid UUID, nil and an error are returned. +func NewUUID(s string) (*UUID, error) { + s = strings.Replace(s, "-", "", -1) + if len(s) != 32 { + return nil, errors.New("bad UUID length != 32") + } + dec, err := hex.DecodeString(s) + if err != nil { + return nil, err + } + var u UUID + for i, b := range dec { + u[i] = b + } + return &u, nil +} + +func (u UUID) Empty() bool { + return reflect.DeepEqual(u, UUID{}) +} + +func (u *UUID) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + uu, err := NewUUID(s) + if uu.Empty() { + return ErrNoEmptyUUID + } + if err == nil { + *u = *uu + } + return err +} + +func (u UUID) MarshalJSON() ([]byte, error) { + if u.Empty() { + return nil, ErrNoEmptyUUID + } + return json.Marshal(u.String()) +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/uuid_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/uuid_test.go new file mode 100644 index 00000000000..59f793c79dd --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/uuid_test.go @@ -0,0 +1,59 @@ +package types + +import "testing" + +func TestNewUUID(t *testing.T) { + tests := []struct { + in string + ws string + }{ + { + "6733C088-A507-4694-AABF-EDBE4FC5266F", + + "6733c088-a507-4694-aabf-edbe4fc5266f", + }, + { + "6733C088A5074694AABFEDBE4FC5266F", + + "6733c088-a507-4694-aabf-edbe4fc5266f", + }, + { + "0aaf0a79-1a39-4d59-abbf-1bebca8209d2", + + "0aaf0a79-1a39-4d59-abbf-1bebca8209d2", + }, + { + "0aaf0a791a394d59abbf1bebca8209d2", + + "0aaf0a79-1a39-4d59-abbf-1bebca8209d2", + }, + } + for i, tt := range tests { + gu, err := NewUUID(tt.in) + if err != nil { + t.Errorf("#%d: err=%v, want %v", i, err, nil) + } + if gs := gu.String(); gs != tt.ws { + t.Errorf("#%d: String()=%v, want %v", i, gs, tt.ws) + } + } +} + +func TestNewUUIDBad(t *testing.T) { + tests := []string{ + "asdf", + "0AAF0A79-1A39-4D59-ABBF-1BEBCA8209D2ABC", + "", + } + for i, tt := range tests { + g, err := NewUUID(tt) + if err == nil { + t.Errorf("#%d: err=nil, want non-nil", i) + } + if g != nil { + t.Errorf("#%d: err=%v, want %v", i, g, nil) + + } + } + +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/volume.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/volume.go new file mode 100644 index 00000000000..a3c92d656a1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/volume.go @@ -0,0 +1,124 @@ +package types + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "path/filepath" + "strconv" + "strings" +) + +// Volume encapsulates a volume which should be mounted into the filesystem +// of all apps in a PodManifest +type Volume struct { + Name ACName `json:"name"` + Kind string `json:"kind"` + + // currently used only by "host" + // TODO(jonboulle): factor out? + Source string `json:"source,omitempty"` + ReadOnly *bool `json:"readOnly,omitempty"` +} + +type volume Volume + +func (v Volume) assertValid() error { + if v.Name.Empty() { + return errors.New("name must be set") + } + + switch v.Kind { + case "empty": + if v.Source != "" { + return errors.New("source for empty volume must be empty") + } + return nil + case "host": + if v.Source == "" { + return errors.New("source for host volume cannot be empty") + } + if !filepath.IsAbs(v.Source) { + return errors.New("source for host volume must be absolute path") + } + return nil + default: + return errors.New(`unrecognized volume kind: should be one of "empty", "host"`) + } +} + +func (v *Volume) UnmarshalJSON(data []byte) error { + var vv volume + if err := json.Unmarshal(data, &vv); err != nil { + return err + } + nv := Volume(vv) + if err := nv.assertValid(); err != nil { + return err + } + *v = nv + return nil +} + +func (v Volume) MarshalJSON() ([]byte, error) { + if err := v.assertValid(); err != nil { + return nil, err + } + return json.Marshal(volume(v)) +} + +func (v Volume) String() string { + s := fmt.Sprintf("%s,kind=%s,readOnly=%t", v.Name, v.Kind, *v.ReadOnly) + if v.Source != "" { + s = s + fmt.Sprintf("source=%s", v.Source) + } + return s +} + +// VolumeFromString takes a command line volume parameter and returns a volume +// +// Example volume parameters: +// database,kind=host,source=/tmp,readOnly=true +func VolumeFromString(vp string) (*Volume, error) { + var vol Volume + + vp = "name=" + vp + v, err := url.ParseQuery(strings.Replace(vp, ",", "&", -1)) + if err != nil { + return nil, err + } + for key, val := range v { + if len(val) > 1 { + return nil, fmt.Errorf("label %s with multiple values %q", key, val) + } + + // TOOD(philips): make this less hardcoded + switch key { + case "name": + acn, err := NewACName(val[0]) + if err != nil { + return nil, err + } + vol.Name = *acn + case "kind": + vol.Kind = val[0] + case "source": + vol.Source = val[0] + case "readOnly": + ro, err := strconv.ParseBool(val[0]) + if err != nil { + return nil, err + } + vol.ReadOnly = &ro + default: + return nil, fmt.Errorf("unknown volume parameter %q", key) + } + } + err = vol.assertValid() + if err != nil { + return nil, err + } + + return &vol, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/types/volume_test.go b/Godeps/_workspace/src/github.com/appc/spec/schema/types/volume_test.go new file mode 100644 index 00000000000..d9674b58b52 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/types/volume_test.go @@ -0,0 +1,85 @@ +package types + +import ( + "reflect" + "testing" +) + +func TestVolumeFromString(t *testing.T) { + trueVar := true + falseVar := false + tests := []struct { + s string + v Volume + }{ + { + "foobar,kind=host,source=/tmp", + Volume{ + Name: "foobar", + Kind: "host", + Source: "/tmp", + ReadOnly: nil, + }, + }, + { + "foobar,kind=host,source=/tmp,readOnly=false", + Volume{ + Name: "foobar", + Kind: "host", + Source: "/tmp", + ReadOnly: &falseVar, + }, + }, + { + "foobar,kind=host,source=/tmp,readOnly=true", + Volume{ + Name: "foobar", + Kind: "host", + Source: "/tmp", + ReadOnly: &trueVar, + }, + }, + { + "foobar,kind=empty", + Volume{ + Name: "foobar", + Kind: "empty", + ReadOnly: nil, + }, + }, + { + "foobar,kind=empty,readOnly=true", + Volume{ + Name: "foobar", + Kind: "empty", + ReadOnly: &trueVar, + }, + }, + } + for i, tt := range tests { + v, err := VolumeFromString(tt.s) + if err != nil { + t.Errorf("#%d: got err=%v, want nil", i, err) + } + if !reflect.DeepEqual(*v, tt.v) { + t.Errorf("#%d: v=%v, want %v", i, *v, tt.v) + } + } +} + +func TestVolumeFromStringBad(t *testing.T) { + tests := []string{ + "#foobar,kind=host,source=/tmp", + "foobar,kind=host,source=/tmp,readOnly=true,asdf=asdf", + "foobar,kind=empty,source=/tmp", + } + for i, in := range tests { + l, err := VolumeFromString(in) + if l != nil { + t.Errorf("#%d: got l=%v, want nil", i, l) + } + if err == nil { + t.Errorf("#%d: got err=nil, want non-nil", i) + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/schema/version.go b/Godeps/_workspace/src/github.com/appc/spec/schema/version.go new file mode 100644 index 00000000000..4f87dca282c --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/schema/version.go @@ -0,0 +1,25 @@ +package schema + +import ( + "github.com/appc/spec/schema/types" +) + +const ( + // version represents the canonical version of the appc spec and tooling. + // For now, the schema and tooling is coupled with the spec itself, so + // this must be kept in sync with the VERSION file in the root of the repo. + version string = "0.5.1+git" +) + +var ( + // AppContainerVersion is the SemVer representation of version + AppContainerVersion types.SemVer +) + +func init() { + v, err := types.NewSemVer(version) + if err != nil { + panic(err) + } + AppContainerVersion = *v +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/.gitignore b/Godeps/_workspace/src/github.com/camlistore/lock/.gitignore new file mode 100644 index 00000000000..b25c15b81fa --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/COPYING b/Godeps/_workspace/src/github.com/camlistore/lock/COPYING new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/README.txt b/Godeps/_workspace/src/github.com/camlistore/lock/README.txt new file mode 100644 index 00000000000..a9eeb33ded2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/README.txt @@ -0,0 +1,3 @@ +File locking library. + +See http://godoc.org/github.com/camlistore/lock diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock.go new file mode 100644 index 00000000000..6268527b04e --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock.go @@ -0,0 +1,158 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sync" +) + +// Lock locks the given file, creating the file if necessary. If the +// file already exists, it must have zero size or an error is returned. +// The lock is an exclusive lock (a write lock), but locked files +// should neither be read from nor written to. Such files should have +// zero size and only exist to co-ordinate ownership across processes. +// +// A nil Closer is returned if an error occurred. Otherwise, close that +// Closer to release the lock. +// +// On Linux, FreeBSD and OSX, a lock has the same semantics as fcntl(2)'s +// advisory locks. In particular, closing any other file descriptor for the +// same file will release the lock prematurely. +// +// Attempting to lock a file that is already locked by the current process +// has undefined behavior. +// +// On other operating systems, lock will fallback to using the presence and +// content of a file named name + '.lock' to implement locking behavior. +func Lock(name string) (io.Closer, error) { + return lockFn(name) +} + +var lockFn = lockPortable + +// Portable version not using fcntl. Doesn't handle crashes as gracefully, +// since it can leave stale lock files. +// TODO: write pid of owner to lock file and on race see if pid is +// still alive? +func lockPortable(name string) (io.Closer, error) { + absName, err := filepath.Abs(name) + if err != nil { + return nil, fmt.Errorf("can't Lock file %q: can't find abs path: %v", name, err) + } + fi, err := os.Stat(absName) + if err == nil && fi.Size() > 0 { + if isStaleLock(absName) { + os.Remove(absName) + } else { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + } + f, err := os.OpenFile(absName, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666) + if err != nil { + return nil, fmt.Errorf("failed to create lock file %s %v", absName, err) + } + if err := json.NewEncoder(f).Encode(&pidLockMeta{OwnerPID: os.Getpid()}); err != nil { + return nil, err + } + return &lockCloser{f: f, abs: absName}, nil +} + +type pidLockMeta struct { + OwnerPID int +} + +func isStaleLock(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + var meta pidLockMeta + if json.NewDecoder(f).Decode(&meta) != nil { + return false + } + if meta.OwnerPID == 0 { + return false + } + p, err := os.FindProcess(meta.OwnerPID) + if err != nil { + // e.g. on Windows + return true + } + // On unix, os.FindProcess always is true, so we have to send + // it a signal to see if it's alive. + if signalZero != nil { + if p.Signal(signalZero) != nil { + return true + } + } + return false +} + +var signalZero os.Signal // nil or set by lock_sigzero.go + +type lockCloser struct { + f *os.File + abs string + once sync.Once + err error +} + +func (lc *lockCloser) Close() error { + lc.once.Do(lc.close) + return lc.err +} + +func (lc *lockCloser) close() { + if err := lc.f.Close(); err != nil { + lc.err = err + } + if err := os.Remove(lc.abs); err != nil { + lc.err = err + } +} + +var ( + lockmu sync.Mutex + locked = map[string]bool{} // abs path -> true +) + +// unlocker is used by the darwin and linux implementations with fcntl +// advisory locks. +type unlocker struct { + f *os.File + abs string +} + +func (u *unlocker) Close() error { + lockmu.Lock() + // Remove is not necessary but it's nice for us to clean up. + // If we do do this, though, it needs to be before the + // u.f.Close below. + os.Remove(u.abs) + if err := u.f.Close(); err != nil { + return err + } + delete(locked, u.abs) + lockmu.Unlock() + return nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_appengine.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_appengine.go new file mode 100644 index 00000000000..ab4cad6ab60 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_appengine.go @@ -0,0 +1,32 @@ +// +build appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "errors" + "io" +) + +func init() { + lockFn = lockAppEngine +} + +func lockAppEngine(name string) (io.Closer, error) { + return nil, errors.New("Lock not available on App Engine") +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_darwin_amd64.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_darwin_amd64.go new file mode 100644 index 00000000000..9fea51fe876 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_darwin_amd64.go @@ -0,0 +1,80 @@ +// +build darwin,amd64 +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err) + } + + // This type matches C's "struct flock" defined in /usr/include/sys/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Start uint64 // sizeof(off_t): 8 + Len uint64 // sizeof(off_t): 8 + Pid uint32 // sizeof(pid_t): 4 + Type uint16 // sizeof(short): 2 + Whence uint16 // sizeof(short): 2 + }{ + Type: syscall.F_WRLCK, + Whence: uint16(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_freebsd.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_freebsd.go new file mode 100644 index 00000000000..d3835d62488 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_freebsd.go @@ -0,0 +1,79 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Start int64 /* off_t starting offset */ + Len int64 /* off_t len = 0 means until end of file */ + Pid int32 /* pid_t lock owner */ + Type int16 /* short lock type: read/write, etc. */ + Whence int16 /* short type of l_start */ + Sysid int32 /* int remote system id or zero for local */ + }{ + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: int32(os.Getpid()), + Type: syscall.F_WRLCK, + Whence: int16(os.SEEK_SET), + Sysid: 0, + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_amd64.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_amd64.go new file mode 100644 index 00000000000..3a7eb00a62e --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_amd64.go @@ -0,0 +1,80 @@ +// +build linux,amd64 +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Type uint32 + Whence uint32 + Start uint64 + Len uint64 + Pid uint32 + }{ + Type: syscall.F_WRLCK, + Whence: uint32(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_arm.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_arm.go new file mode 100644 index 00000000000..c2a0a102e96 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_arm.go @@ -0,0 +1,81 @@ +// +build linux,arm +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Type uint16 + Whence uint16 + Start uint32 + Len uint32 + Pid uint32 + }{ + Type: syscall.F_WRLCK, + Whence: uint16(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + const F_SETLK = 6 // actual value. syscall package is wrong: golang.org/issue/7059 + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_plan9.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_plan9.go new file mode 100644 index 00000000000..bdf4e229284 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_plan9.go @@ -0,0 +1,55 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +func init() { + lockFn = lockPlan9 +} + +func lockPlan9(name string) (io.Closer, error) { + var f *os.File + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644) + if err != nil { + return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err) + } + + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_sigzero.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_sigzero.go new file mode 100644 index 00000000000..fd3ba2db197 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_sigzero.go @@ -0,0 +1,26 @@ +// +build !appengine +// +build linux darwin freebsd openbsd netbsd dragonfly + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import "syscall" + +func init() { + signalZero = syscall.Signal(0) +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_test.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_test.go new file mode 100644 index 00000000000..518d2f025fe --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" +) + +func TestLock(t *testing.T) { + testLock(t, false) +} + +func TestLockPortable(t *testing.T) { + testLock(t, true) +} + +func TestLockInChild(t *testing.T) { + f := os.Getenv("TEST_LOCK_FILE") + if f == "" { + // not child + return + } + lock := Lock + if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_PORTABLE")); v { + lock = lockPortable + } + + lk, err := lock(f) + if err != nil { + log.Fatalf("Lock failed: %v", err) + } + + if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_CRASH")); v { + // Simulate a crash, or at least not unlocking the + // lock. We still exit 0 just to simplify the parent + // process exec code. + os.Exit(0) + } + lk.Close() +} + +func testLock(t *testing.T, portable bool) { + lock := Lock + if portable { + lock = lockPortable + } + + td, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(td) + + path := filepath.Join(td, "foo.lock") + + childLock := func(crash bool) error { + cmd := exec.Command(os.Args[0], "-test.run=LockInChild$") + cmd.Env = []string{"TEST_LOCK_FILE=" + path} + if portable { + cmd.Env = append(cmd.Env, "TEST_LOCK_PORTABLE=1") + } + if crash { + cmd.Env = append(cmd.Env, "TEST_LOCK_CRASH=1") + } + out, err := cmd.CombinedOutput() + t.Logf("Child output: %q (err %v)", out, err) + if err != nil { + return fmt.Errorf("Child Process lock of %s failed: %v %s", path, err, out) + } + return nil + } + + t.Logf("Locking in crashing child...") + if err := childLock(true); err != nil { + t.Fatalf("first lock in child process: %v", err) + } + + t.Logf("Locking+unlocking in child...") + if err := childLock(false); err != nil { + t.Fatalf("lock in child process after crashing child: %v", err) + } + + t.Logf("Locking in parent...") + lk1, err := lock(path) + if err != nil { + t.Fatal(err) + } + + t.Logf("Again in parent...") + _, err = lock(path) + if err == nil { + t.Fatal("expected second lock to fail") + } + + t.Logf("Locking in child...") + if childLock(false) == nil { + t.Fatalf("expected lock in child process to fail") + } + + t.Logf("Unlocking lock in parent") + if err := lk1.Close(); err != nil { + t.Fatal(err) + } + + lk3, err := lock(path) + if err != nil { + t.Fatal(err) + } + lk3.Close() +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/doc.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/doc.go new file mode 100644 index 00000000000..fa0158020e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/doc.go @@ -0,0 +1,19 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package httptypes defines how etcd's HTTP API entities are serialized to and deserialized from JSON. +*/ + +package httptypes diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors.go new file mode 100644 index 00000000000..7e0d275ebbb --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors.go @@ -0,0 +1,49 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "encoding/json" + "log" + "net/http" +) + +type HTTPError struct { + Message string `json:"message"` + // HTTP return code + Code int `json:"-"` +} + +func (e HTTPError) Error() string { + return e.Message +} + +// TODO(xiangli): handle http write errors +func (e HTTPError) WriteTo(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(e.Code) + b, err := json.Marshal(e) + if err != nil { + log.Panicf("marshal HTTPError should never fail: %v", err) + } + w.Write(b) +} + +func NewHTTPError(code int, m string) *HTTPError { + return &HTTPError{ + Message: m, + Code: code, + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors_test.go new file mode 100644 index 00000000000..f5cec6d4579 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors_test.go @@ -0,0 +1,47 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestHTTPErrorWriteTo(t *testing.T) { + err := NewHTTPError(http.StatusBadRequest, "what a bad request you made!") + rr := httptest.NewRecorder() + err.WriteTo(rr) + + wcode := http.StatusBadRequest + wheader := http.Header(map[string][]string{ + "Content-Type": []string{"application/json"}, + }) + wbody := `{"message":"what a bad request you made!"}` + + if wcode != rr.Code { + t.Errorf("HTTP status code %d, want %d", rr.Code, wcode) + } + + if !reflect.DeepEqual(wheader, rr.HeaderMap) { + t.Errorf("HTTP headers %v, want %v", rr.HeaderMap, wheader) + } + + gbody := rr.Body.String() + if wbody != gbody { + t.Errorf("HTTP body %q, want %q", gbody, wbody) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member.go new file mode 100644 index 00000000000..30ecbb53939 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member.go @@ -0,0 +1,67 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "encoding/json" + + "github.com/coreos/etcd/pkg/types" +) + +type Member struct { + ID string `json:"id"` + Name string `json:"name"` + PeerURLs []string `json:"peerURLs"` + ClientURLs []string `json:"clientURLs"` +} + +type MemberCreateRequest struct { + PeerURLs types.URLs +} + +type MemberUpdateRequest struct { + MemberCreateRequest +} + +func (m *MemberCreateRequest) UnmarshalJSON(data []byte) error { + s := struct { + PeerURLs []string `json:"peerURLs"` + }{} + + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + urls, err := types.NewURLs(s.PeerURLs) + if err != nil { + return err + } + + m.PeerURLs = urls + return nil +} + +type MemberCollection []Member + +func (c *MemberCollection) MarshalJSON() ([]byte, error) { + d := struct { + Members []Member `json:"members"` + }{ + Members: []Member(*c), + } + + return json.Marshal(d) +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member_test.go new file mode 100644 index 00000000000..e0b29d88311 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member_test.go @@ -0,0 +1,135 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "encoding/json" + "net/url" + "reflect" + "testing" + + "github.com/coreos/etcd/pkg/types" +) + +func TestMemberUnmarshal(t *testing.T) { + tests := []struct { + body []byte + wantMember Member + wantError bool + }{ + // no URLs, just check ID & Name + { + body: []byte(`{"id": "c", "name": "dungarees"}`), + wantMember: Member{ID: "c", Name: "dungarees", PeerURLs: nil, ClientURLs: nil}, + }, + + // both client and peer URLs + { + body: []byte(`{"peerURLs": ["http://127.0.0.1:4001"], "clientURLs": ["http://127.0.0.1:4001"]}`), + wantMember: Member{ + PeerURLs: []string{ + "http://127.0.0.1:4001", + }, + ClientURLs: []string{ + "http://127.0.0.1:4001", + }, + }, + }, + + // multiple peer URLs + { + body: []byte(`{"peerURLs": ["http://127.0.0.1:4001", "https://example.com"]}`), + wantMember: Member{ + PeerURLs: []string{ + "http://127.0.0.1:4001", + "https://example.com", + }, + ClientURLs: nil, + }, + }, + + // multiple client URLs + { + body: []byte(`{"clientURLs": ["http://127.0.0.1:4001", "https://example.com"]}`), + wantMember: Member{ + PeerURLs: nil, + ClientURLs: []string{ + "http://127.0.0.1:4001", + "https://example.com", + }, + }, + }, + + // invalid JSON + { + body: []byte(`{"peerU`), + wantError: true, + }, + } + + for i, tt := range tests { + got := Member{} + err := json.Unmarshal(tt.body, &got) + if tt.wantError != (err != nil) { + t.Errorf("#%d: want error %t, got %v", i, tt.wantError, err) + continue + } + + if !reflect.DeepEqual(tt.wantMember, got) { + t.Errorf("#%d: incorrect output: want=%#v, got=%#v", i, tt.wantMember, got) + } + } +} + +func TestMemberCreateRequestUnmarshal(t *testing.T) { + body := []byte(`{"peerURLs": ["http://127.0.0.1:8081", "https://127.0.0.1:8080"]}`) + want := MemberCreateRequest{ + PeerURLs: types.URLs([]url.URL{ + url.URL{Scheme: "http", Host: "127.0.0.1:8081"}, + url.URL{Scheme: "https", Host: "127.0.0.1:8080"}, + }), + } + + var req MemberCreateRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Fatalf("Unmarshal returned unexpected err=%v", err) + } + + if !reflect.DeepEqual(want, req) { + t.Fatalf("Failed to unmarshal MemberCreateRequest: want=%#v, got=%#v", want, req) + } +} + +func TestMemberCreateRequestUnmarshalFail(t *testing.T) { + tests := [][]byte{ + // invalid JSON + []byte(``), + []byte(`{`), + + // spot-check validation done in types.NewURLs + []byte(`{"peerURLs": "foo"}`), + []byte(`{"peerURLs": ["."]}`), + []byte(`{"peerURLs": []}`), + []byte(`{"peerURLs": ["http://127.0.0.1:4001/foo"]}`), + []byte(`{"peerURLs": ["http://127.0.0.1"]}`), + } + + for i, tt := range tests { + var req MemberCreateRequest + if err := json.Unmarshal(tt, &req); err == nil { + t.Errorf("#%d: expected err, got nil", i) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id.go new file mode 100644 index 00000000000..88cb9e63494 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id.go @@ -0,0 +1,41 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "strconv" +) + +// ID represents a generic identifier which is canonically +// stored as a uint64 but is typically represented as a +// base-16 string for input/output +type ID uint64 + +func (i ID) String() string { + return strconv.FormatUint(uint64(i), 16) +} + +// IDFromString attempts to create an ID from a base-16 string. +func IDFromString(s string) (ID, error) { + i, err := strconv.ParseUint(s, 16, 64) + return ID(i), err +} + +// IDSlice implements the sort interface +type IDSlice []ID + +func (p IDSlice) Len() int { return len(p) } +func (p IDSlice) Less(i, j int) bool { return uint64(p[i]) < uint64(p[j]) } +func (p IDSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id_test.go new file mode 100644 index 00000000000..97d168f58e2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id_test.go @@ -0,0 +1,95 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestIDString(t *testing.T) { + tests := []struct { + input ID + want string + }{ + { + input: 12, + want: "c", + }, + { + input: 4918257920282737594, + want: "444129853c343bba", + }, + } + + for i, tt := range tests { + got := tt.input.String() + if tt.want != got { + t.Errorf("#%d: ID.String failure: want=%v, got=%v", i, tt.want, got) + } + } +} + +func TestIDFromString(t *testing.T) { + tests := []struct { + input string + want ID + }{ + { + input: "17", + want: 23, + }, + { + input: "612840dae127353", + want: 437557308098245459, + }, + } + + for i, tt := range tests { + got, err := IDFromString(tt.input) + if err != nil { + t.Errorf("#%d: IDFromString failure: err=%v", i, err) + continue + } + if tt.want != got { + t.Errorf("#%d: IDFromString failure: want=%v, got=%v", i, tt.want, got) + } + } +} + +func TestIDFromStringFail(t *testing.T) { + tests := []string{ + "", + "XXX", + "612840dae127353612840dae127353", + } + + for i, tt := range tests { + _, err := IDFromString(tt) + if err == nil { + t.Fatalf("#%d: IDFromString expected error, but err=nil", i) + } + } +} + +func TestIDSlice(t *testing.T) { + g := []ID{10, 500, 5, 1, 100, 25} + w := []ID{1, 5, 10, 25, 100, 500} + sort.Sort(IDSlice(g)) + if !reflect.DeepEqual(g, w) { + t.Errorf("slice after sort = %#v, want %#v", g, w) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set.go new file mode 100644 index 00000000000..32287522b11 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set.go @@ -0,0 +1,178 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "sync" +) + +type Set interface { + Add(string) + Remove(string) + Contains(string) bool + Equals(Set) bool + Length() int + Values() []string + Copy() Set + Sub(Set) Set +} + +func NewUnsafeSet(values ...string) *unsafeSet { + set := &unsafeSet{make(map[string]struct{})} + for _, v := range values { + set.Add(v) + } + return set +} + +func NewThreadsafeSet(values ...string) *tsafeSet { + us := NewUnsafeSet(values...) + return &tsafeSet{us, sync.RWMutex{}} +} + +type unsafeSet struct { + d map[string]struct{} +} + +// Add adds a new value to the set (no-op if the value is already present) +func (us *unsafeSet) Add(value string) { + us.d[value] = struct{}{} +} + +// Remove removes the given value from the set +func (us *unsafeSet) Remove(value string) { + delete(us.d, value) +} + +// Contains returns whether the set contains the given value +func (us *unsafeSet) Contains(value string) (exists bool) { + _, exists = us.d[value] + return +} + +// ContainsAll returns whether the set contains all given values +func (us *unsafeSet) ContainsAll(values []string) bool { + for _, s := range values { + if !us.Contains(s) { + return false + } + } + return true +} + +// Equals returns whether the contents of two sets are identical +func (us *unsafeSet) Equals(other Set) bool { + v1 := sort.StringSlice(us.Values()) + v2 := sort.StringSlice(other.Values()) + v1.Sort() + v2.Sort() + return reflect.DeepEqual(v1, v2) +} + +// Length returns the number of elements in the set +func (us *unsafeSet) Length() int { + return len(us.d) +} + +// Values returns the values of the Set in an unspecified order. +func (us *unsafeSet) Values() (values []string) { + values = make([]string, 0) + for val, _ := range us.d { + values = append(values, val) + } + return +} + +// Copy creates a new Set containing the values of the first +func (us *unsafeSet) Copy() Set { + cp := NewUnsafeSet() + for val, _ := range us.d { + cp.Add(val) + } + + return cp +} + +// Sub removes all elements in other from the set +func (us *unsafeSet) Sub(other Set) Set { + oValues := other.Values() + result := us.Copy().(*unsafeSet) + + for _, val := range oValues { + if _, ok := result.d[val]; !ok { + continue + } + delete(result.d, val) + } + + return result +} + +type tsafeSet struct { + us *unsafeSet + m sync.RWMutex +} + +func (ts *tsafeSet) Add(value string) { + ts.m.Lock() + defer ts.m.Unlock() + ts.us.Add(value) +} + +func (ts *tsafeSet) Remove(value string) { + ts.m.Lock() + defer ts.m.Unlock() + ts.us.Remove(value) +} + +func (ts *tsafeSet) Contains(value string) (exists bool) { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Contains(value) +} + +func (ts *tsafeSet) Equals(other Set) bool { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Equals(other) +} + +func (ts *tsafeSet) Length() int { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Length() +} + +func (ts *tsafeSet) Values() (values []string) { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Values() +} + +func (ts *tsafeSet) Copy() Set { + ts.m.RLock() + defer ts.m.RUnlock() + usResult := ts.us.Copy().(*unsafeSet) + return &tsafeSet{usResult, sync.RWMutex{}} +} + +func (ts *tsafeSet) Sub(other Set) Set { + ts.m.RLock() + defer ts.m.RUnlock() + usResult := ts.us.Sub(other).(*unsafeSet) + return &tsafeSet{usResult, sync.RWMutex{}} +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set_test.go new file mode 100644 index 00000000000..ff1ecc68d3c --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set_test.go @@ -0,0 +1,186 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestUnsafeSet(t *testing.T) { + driveSetTests(t, NewUnsafeSet()) +} + +func TestThreadsafeSet(t *testing.T) { + driveSetTests(t, NewThreadsafeSet()) +} + +// Check that two slices contents are equal; order is irrelevant +func equal(a, b []string) bool { + as := sort.StringSlice(a) + bs := sort.StringSlice(b) + as.Sort() + bs.Sort() + return reflect.DeepEqual(as, bs) +} + +func driveSetTests(t *testing.T, s Set) { + // Verify operations on an empty set + eValues := []string{} + values := s.Values() + if !reflect.DeepEqual(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + if l := s.Length(); l != 0 { + t.Fatalf("Expected length=0, got %d", l) + } + for _, v := range []string{"foo", "bar", "baz"} { + if s.Contains(v) { + t.Fatalf("Expect s.Contains(%q) to be fale, got true", v) + } + } + + // Add three items, ensure they show up + s.Add("foo") + s.Add("bar") + s.Add("baz") + + eValues = []string{"foo", "bar", "baz"} + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + + for _, v := range eValues { + if !s.Contains(v) { + t.Fatalf("Expect s.Contains(%q) to be true, got false", v) + } + } + + if l := s.Length(); l != 3 { + t.Fatalf("Expected length=3, got %d", l) + } + + // Add the same item a second time, ensuring it is not duplicated + s.Add("foo") + + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + if l := s.Length(); l != 3 { + t.Fatalf("Expected length=3, got %d", l) + } + + // Remove all items, ensure they are gone + s.Remove("foo") + s.Remove("bar") + s.Remove("baz") + + eValues = []string{} + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + + if l := s.Length(); l != 0 { + t.Fatalf("Expected length=0, got %d", l) + } + + // Create new copies of the set, and ensure they are unlinked to the + // original Set by making modifications + s.Add("foo") + s.Add("bar") + cp1 := s.Copy() + cp2 := s.Copy() + s.Remove("foo") + cp3 := s.Copy() + cp1.Add("baz") + + for i, tt := range []struct { + want []string + got []string + }{ + {[]string{"bar"}, s.Values()}, + {[]string{"foo", "bar", "baz"}, cp1.Values()}, + {[]string{"foo", "bar"}, cp2.Values()}, + {[]string{"bar"}, cp3.Values()}, + } { + if !equal(tt.want, tt.got) { + t.Fatalf("case %d: expect values=%v got %v", i, tt.want, tt.got) + } + } + + for i, tt := range []struct { + want bool + got bool + }{ + {true, s.Equals(cp3)}, + {true, cp3.Equals(s)}, + {false, s.Equals(cp2)}, + {false, s.Equals(cp1)}, + {false, cp1.Equals(s)}, + {false, cp2.Equals(s)}, + {false, cp2.Equals(cp1)}, + } { + if tt.got != tt.want { + t.Fatalf("case %d: want %t, got %t", i, tt.want, tt.got) + + } + } + + // Subtract values from a Set, ensuring a new Set is created and + // the original Sets are unmodified + sub1 := cp1.Sub(s) + sub2 := cp2.Sub(cp1) + + for i, tt := range []struct { + want []string + got []string + }{ + {[]string{"foo", "bar", "baz"}, cp1.Values()}, + {[]string{"foo", "bar"}, cp2.Values()}, + {[]string{"bar"}, s.Values()}, + {[]string{"foo", "baz"}, sub1.Values()}, + {[]string{}, sub2.Values()}, + } { + if !equal(tt.want, tt.got) { + t.Fatalf("case %d: expect values=%v got %v", i, tt.want, tt.got) + } + } +} + +func TestUnsafeSetContainsAll(t *testing.T) { + vals := []string{"foo", "bar", "baz"} + s := NewUnsafeSet(vals...) + + tests := []struct { + strs []string + wcontain bool + }{ + {[]string{}, true}, + {vals[:1], true}, + {vals[:2], true}, + {vals, true}, + {[]string{"cuz"}, false}, + {[]string{vals[0], "cuz"}, false}, + } + for i, tt := range tests { + if g := s.ContainsAll(tt.strs); g != tt.wcontain { + t.Errorf("#%d: ok = %v, want %v", i, g, tt.wcontain) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice.go new file mode 100644 index 00000000000..0327950f706 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice.go @@ -0,0 +1,22 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +// Uint64Slice implements sort interface +type Uint64Slice []uint64 + +func (p Uint64Slice) Len() int { return len(p) } +func (p Uint64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice_test.go new file mode 100644 index 00000000000..95e37e04d20 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice_test.go @@ -0,0 +1,30 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestUint64Slice(t *testing.T) { + g := Uint64Slice{10, 500, 5, 1, 100, 25} + w := Uint64Slice{1, 5, 10, 25, 100, 500} + sort.Sort(g) + if !reflect.DeepEqual(g, w) { + t.Errorf("slice after sort = %#v, want %#v", g, w) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls.go new file mode 100644 index 00000000000..ce2483ffaaa --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls.go @@ -0,0 +1,74 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "errors" + "fmt" + "net" + "net/url" + "sort" + "strings" +) + +type URLs []url.URL + +func NewURLs(strs []string) (URLs, error) { + all := make([]url.URL, len(strs)) + if len(all) == 0 { + return nil, errors.New("no valid URLs given") + } + for i, in := range strs { + in = strings.TrimSpace(in) + u, err := url.Parse(in) + if err != nil { + return nil, err + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("URL scheme must be http or https: %s", in) + } + if _, _, err := net.SplitHostPort(u.Host); err != nil { + return nil, fmt.Errorf(`URL address does not have the form "host:port": %s`, in) + } + if u.Path != "" { + return nil, fmt.Errorf("URL must not contain a path: %s", in) + } + all[i] = *u + } + us := URLs(all) + us.Sort() + + return us, nil +} + +func (us URLs) String() string { + return strings.Join(us.StringSlice(), ",") +} + +func (us *URLs) Sort() { + sort.Sort(us) +} +func (us URLs) Len() int { return len(us) } +func (us URLs) Less(i, j int) bool { return us[i].String() < us[j].String() } +func (us URLs) Swap(i, j int) { us[i], us[j] = us[j], us[i] } + +func (us URLs) StringSlice() []string { + out := make([]string, len(us)) + for i := range us { + out[i] = us[i].String() + } + + return out +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls_test.go new file mode 100644 index 00000000000..41caa5d68a5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls_test.go @@ -0,0 +1,169 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "testing" + + "github.com/coreos/etcd/pkg/testutil" +) + +func TestNewURLs(t *testing.T) { + tests := []struct { + strs []string + wurls URLs + }{ + { + []string{"http://127.0.0.1:4001"}, + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + }, + // it can trim space + { + []string{" http://127.0.0.1:4001 "}, + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + }, + // it does sort + { + []string{ + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + }, + testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + }), + }, + } + for i, tt := range tests { + urls, _ := NewURLs(tt.strs) + if !reflect.DeepEqual(urls, tt.wurls) { + t.Errorf("#%d: urls = %+v, want %+v", i, urls, tt.wurls) + } + } +} + +func TestURLsString(t *testing.T) { + tests := []struct { + us URLs + wstr string + }{ + { + URLs{}, + "", + }, + { + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + "http://127.0.0.1:4001", + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + }), + "http://127.0.0.1:4001,http://127.0.0.2:4001", + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + }), + "http://127.0.0.2:4001,http://127.0.0.1:4001", + }, + } + for i, tt := range tests { + g := tt.us.String() + if g != tt.wstr { + t.Errorf("#%d: string = %s, want %s", i, g, tt.wstr) + } + } +} + +func TestURLsSort(t *testing.T) { + g := testutil.MustNewURLs(t, []string{ + "http://127.0.0.4:4001", + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + "http://127.0.0.3:4001", + }) + w := testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + "http://127.0.0.3:4001", + "http://127.0.0.4:4001", + }) + gurls := URLs(g) + gurls.Sort() + if !reflect.DeepEqual(g, w) { + t.Errorf("URLs after sort = %#v, want %#v", g, w) + } +} + +func TestURLsStringSlice(t *testing.T) { + tests := []struct { + us URLs + wstr []string + }{ + { + URLs{}, + []string{}, + }, + { + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + []string{"http://127.0.0.1:4001"}, + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + }), + []string{"http://127.0.0.1:4001", "http://127.0.0.2:4001"}, + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + }), + []string{"http://127.0.0.2:4001", "http://127.0.0.1:4001"}, + }, + } + for i, tt := range tests { + g := tt.us.StringSlice() + if !reflect.DeepEqual(g, tt.wstr) { + t.Errorf("#%d: string slice = %+v, want %+v", i, g, tt.wstr) + } + } +} + +func TestNewURLsFail(t *testing.T) { + tests := [][]string{ + // no urls given + {}, + // missing protocol scheme + {"://127.0.0.1:4001"}, + // unsupported scheme + {"mailto://127.0.0.1:4001"}, + // not conform to host:port + {"http://127.0.0.1"}, + // contain a path + {"http://127.0.0.1:4001/path"}, + } + for i, tt := range tests { + _, err := NewURLs(tt) + if err == nil { + t.Errorf("#%d: err = nil, but error", i) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver.go b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver.go new file mode 100644 index 00000000000..4e10221a182 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver.go @@ -0,0 +1,202 @@ +package semver + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" +) + +type Version struct { + Major int64 + Minor int64 + Patch int64 + PreRelease PreRelease + Metadata string +} + +type PreRelease string + +func splitOff(input *string, delim string) (val string) { + parts := strings.SplitN(*input, delim, 2) + + if len(parts) == 2 { + *input = parts[0] + val = parts[1] + } + + return val +} + +func NewVersion(version string) (*Version, error) { + v := Version{} + + dotParts := strings.SplitN(version, ".", 3) + + if len(dotParts) != 3 { + return nil, errors.New(fmt.Sprintf("%s is not in dotted-tri format", version)) + } + + v.Metadata = splitOff(&dotParts[2], "+") + v.PreRelease = PreRelease(splitOff(&dotParts[2], "-")) + + parsed := make([]int64, 3, 3) + + for i, v := range dotParts[:3] { + val, err := strconv.ParseInt(v, 10, 64) + parsed[i] = val + if err != nil { + return nil, err + } + } + + v.Major = parsed[0] + v.Minor = parsed[1] + v.Patch = parsed[2] + + return &v, nil +} + +func (v *Version) String() string { + var buffer bytes.Buffer + + base := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) + buffer.WriteString(base) + + if v.PreRelease != "" { + buffer.WriteString(fmt.Sprintf("-%s", v.PreRelease)) + } + + if v.Metadata != "" { + buffer.WriteString(fmt.Sprintf("+%s", v.Metadata)) + } + + return buffer.String() +} + +func (v *Version) LessThan(versionB Version) bool { + versionA := *v + cmp := recursiveCompare(versionA.Slice(), versionB.Slice()) + + if cmp == 0 { + cmp = preReleaseCompare(versionA, versionB) + } + + if cmp == -1 { + return true + } + + return false +} + +/* Slice converts the comparable parts of the semver into a slice of strings */ +func (v *Version) Slice() []int64 { + return []int64{v.Major, v.Minor, v.Patch} +} + +func (p *PreRelease) Slice() []string { + preRelease := string(*p) + return strings.Split(preRelease, ".") +} + +func preReleaseCompare(versionA Version, versionB Version) int { + a := versionA.PreRelease + b := versionB.PreRelease + + /* Handle the case where if two versions are otherwise equal it is the + * one without a PreRelease that is greater */ + if len(a) == 0 && (len(b) > 0) { + return 1 + } else if len(b) == 0 && (len(a) > 0) { + return -1 + } + + // If there is a prelease, check and compare each part. + return recursivePreReleaseCompare(a.Slice(), b.Slice()) +} + +func recursiveCompare(versionA []int64, versionB []int64) int { + if len(versionA) == 0 { + return 0 + } + + a := versionA[0] + b := versionB[0] + + if a > b { + return 1 + } else if a < b { + return -1 + } + + return recursiveCompare(versionA[1:], versionB[1:]) +} + +func recursivePreReleaseCompare(versionA []string, versionB []string) int { + // Handle slice length disparity. + if len(versionA) == 0 { + // Nothing to compare too, so we return 0 + return 0 + } else if len(versionB) == 0 { + // We're longer than versionB so return 1. + return 1 + } + + a := versionA[0] + b := versionB[0] + + aInt := false; bInt := false + + aI, err := strconv.Atoi(versionA[0]) + if err == nil { + aInt = true + } + + bI, err := strconv.Atoi(versionB[0]) + if err == nil { + bInt = true + } + + // Handle Integer Comparison + if aInt && bInt { + if aI > bI { + return 1 + } else if aI < bI { + return -1 + } + } + + // Handle String Comparison + if a > b { + return 1 + } else if a < b { + return -1 + } + + return recursivePreReleaseCompare(versionA[1:], versionB[1:]) +} + +// BumpMajor increments the Major field by 1 and resets all other fields to their default values +func (v *Version) BumpMajor() { + v.Major += 1 + v.Minor = 0 + v.Patch = 0 + v.PreRelease = PreRelease("") + v.Metadata = "" +} + +// BumpMinor increments the Minor field by 1 and resets all other fields to their default values +func (v *Version) BumpMinor() { + v.Minor += 1 + v.Patch = 0 + v.PreRelease = PreRelease("") + v.Metadata = "" +} + +// BumpPatch increments the Patch field by 1 and resets all other fields to their default values +func (v *Version) BumpPatch() { + v.Patch += 1 + v.PreRelease = PreRelease("") + v.Metadata = "" +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver_test.go b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver_test.go new file mode 100644 index 00000000000..de09cd7dc0c --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/semver_test.go @@ -0,0 +1,187 @@ +package semver + +import ( + "math/rand" + "testing" + "time" +) + +type fixture struct { + greaterVersion string + lesserVersion string +} + +var fixtures = []fixture{ + fixture{"0.0.0", "0.0.0-foo"}, + fixture{"0.0.1", "0.0.0"}, + fixture{"1.0.0", "0.9.9"}, + fixture{"0.10.0", "0.9.0"}, + fixture{"0.99.0", "0.10.0"}, + fixture{"2.0.0", "1.2.3"}, + fixture{"0.0.0", "0.0.0-foo"}, + fixture{"0.0.1", "0.0.0"}, + fixture{"1.0.0", "0.9.9"}, + fixture{"0.10.0", "0.9.0"}, + fixture{"0.99.0", "0.10.0"}, + fixture{"2.0.0", "1.2.3"}, + fixture{"0.0.0", "0.0.0-foo"}, + fixture{"0.0.1", "0.0.0"}, + fixture{"1.0.0", "0.9.9"}, + fixture{"0.10.0", "0.9.0"}, + fixture{"0.99.0", "0.10.0"}, + fixture{"2.0.0", "1.2.3"}, + fixture{"1.2.3", "1.2.3-asdf"}, + fixture{"1.2.3", "1.2.3-4"}, + fixture{"1.2.3", "1.2.3-4-foo"}, + fixture{"1.2.3-5-foo", "1.2.3-5"}, + fixture{"1.2.3-5", "1.2.3-4"}, + fixture{"1.2.3-5-foo", "1.2.3-5-Foo"}, + fixture{"3.0.0", "2.7.2+asdf"}, + fixture{"3.0.0+foobar", "2.7.2"}, + fixture{"1.2.3-a.10", "1.2.3-a.5"}, + fixture{"1.2.3-a.b", "1.2.3-a.5"}, + fixture{"1.2.3-a.b", "1.2.3-a"}, + fixture{"1.2.3-a.b.c.10.d.5", "1.2.3-a.b.c.5.d.100"}, + fixture{"1.0.0", "1.0.0-rc.1"}, + fixture{"1.0.0-rc.2", "1.0.0-rc.1"}, + fixture{"1.0.0-rc.1", "1.0.0-beta.11"}, + fixture{"1.0.0-beta.11", "1.0.0-beta.2"}, + fixture{"1.0.0-beta.2", "1.0.0-beta"}, + fixture{"1.0.0-beta", "1.0.0-alpha.beta"}, + fixture{"1.0.0-alpha.beta", "1.0.0-alpha.1"}, + fixture{"1.0.0-alpha.1", "1.0.0-alpha"}, +} + +func TestCompare(t *testing.T) { + for _, v := range fixtures { + gt, err := NewVersion(v.greaterVersion) + if err != nil { + t.Error(err) + } + + lt, err := NewVersion(v.lesserVersion) + if err != nil { + t.Error(err) + } + + if gt.LessThan(*lt) == true { + t.Errorf("%s should not be less than %s", gt, lt) + } + } +} + +func testString(t *testing.T, orig string, version *Version) { + if orig != version.String() { + t.Errorf("%s != %s", orig, version) + } +} + +func TestString(t *testing.T) { + for _, v := range fixtures { + gt, err := NewVersion(v.greaterVersion) + if err != nil { + t.Error(err) + } + testString(t, v.greaterVersion, gt) + + lt, err := NewVersion(v.lesserVersion) + if err != nil { + t.Error(err) + } + testString(t, v.lesserVersion, lt) + } +} + +func shuffleStringSlice(src []string) []string { + dest := make([]string, len(src)) + rand.Seed(time.Now().Unix()) + perm := rand.Perm(len(src)) + for i, v := range perm { + dest[v] = src[i] + } + return dest +} + +func TestSort(t *testing.T) { + sortedVersions := []string{"1.0.0", "1.0.2", "1.2.0", "3.1.1"} + unsortedVersions := shuffleStringSlice(sortedVersions) + + semvers := []*Version{} + for _, v := range unsortedVersions { + sv, err := NewVersion(v) + if err != nil { + t.Fatal(err) + } + semvers = append(semvers, sv) + } + + Sort(semvers) + + for idx, sv := range semvers { + if sv.String() != sortedVersions[idx] { + t.Fatalf("incorrect sort at index %v", idx) + } + } +} + +func TestBumpMajor(t *testing.T) { + version, _ := NewVersion("1.0.0") + version.BumpMajor() + if version.Major != 2 { + t.Fatalf("bumping major on 1.0.0 resulted in %v", version) + } + + version, _ = NewVersion("1.5.2") + version.BumpMajor() + if version.Minor != 0 && version.Patch != 0 { + t.Fatalf("bumping major on 1.5.2 resulted in %v", version) + } + + version, _ = NewVersion("1.0.0+build.1-alpha.1") + version.BumpMajor() + if version.PreRelease != "" && version.PreRelease != "" { + t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version) + } +} + +func TestBumpMinor(t *testing.T) { + version, _ := NewVersion("1.0.0") + version.BumpMinor() + + if version.Major != 1 { + t.Fatalf("bumping minor on 1.0.0 resulted in %v", version) + } + + if version.Minor != 1 { + t.Fatalf("bumping major on 1.0.0 resulted in %v", version) + } + + version, _ = NewVersion("1.0.0+build.1-alpha.1") + version.BumpMinor() + if version.PreRelease != "" && version.PreRelease != "" { + t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version) + } +} + +func TestBumpPatch(t *testing.T) { + version, _ := NewVersion("1.0.0") + version.BumpPatch() + + if version.Major != 1 { + t.Fatalf("bumping minor on 1.0.0 resulted in %v", version) + } + + if version.Minor != 0 { + t.Fatalf("bumping major on 1.0.0 resulted in %v", version) + } + + if version.Patch != 1 { + t.Fatalf("bumping major on 1.0.0 resulted in %v", version) + } + + version, _ = NewVersion("1.0.0+build.1-alpha.1") + version.BumpPatch() + if version.PreRelease != "" && version.PreRelease != "" { + t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-semver/semver/sort.go b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/sort.go new file mode 100644 index 00000000000..86203007ae9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-semver/semver/sort.go @@ -0,0 +1,24 @@ +package semver + +import ( + "sort" +) + +type Versions []*Version + +func (s Versions) Len() int { + return len(s) +} + +func (s Versions) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s Versions) Less(i, j int) bool { + return s[i].LessThan(*s[j]) +} + +// Sort sorts the given slice of Version +func Sort(versions []*Version) { + sort.Sort(Versions(versions)) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/deserialize.go b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/deserialize.go new file mode 100644 index 00000000000..2b1522e2e5d --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/deserialize.go @@ -0,0 +1,213 @@ +package unit + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "strings" + "unicode" +) + +// Deserialize parses a systemd unit file into a list of UnitOption objects. +func Deserialize(f io.Reader) (opts []*UnitOption, err error) { + lexer, optchan, errchan := newLexer(f) + go lexer.lex() + + for opt := range optchan { + opts = append(opts, &(*opt)) + } + + err = <-errchan + return opts, err +} + +func newLexer(f io.Reader) (*lexer, <-chan *UnitOption, <-chan error) { + optchan := make(chan *UnitOption) + errchan := make(chan error, 1) + buf := bufio.NewReader(f) + + return &lexer{buf, optchan, errchan, ""}, optchan, errchan +} + +type lexer struct { + buf *bufio.Reader + optchan chan *UnitOption + errchan chan error + section string +} + +func (l *lexer) lex() { + var err error + next := l.lexNextSection + for next != nil { + next, err = next() + if err != nil { + l.errchan <- err + break + } + } + + close(l.optchan) + close(l.errchan) +} + +type lexStep func() (lexStep, error) + +func (l *lexer) lexSectionName() (lexStep, error) { + sec, err := l.buf.ReadBytes(']') + if err != nil { + return nil, errors.New("unable to find end of section") + } + + return l.lexSectionSuffixFunc(string(sec[:len(sec)-1])), nil +} + +func (l *lexer) lexSectionSuffixFunc(section string) lexStep { + return func() (lexStep, error) { + garbage, err := l.toEOL() + if err != nil { + return nil, err + } + + garbage = bytes.TrimSpace(garbage) + if len(garbage) > 0 { + return nil, fmt.Errorf("found garbage after section name %s: %v", l.section, garbage) + } + + return l.lexNextSectionOrOptionFunc(section), nil + } +} + +func (l *lexer) ignoreLineFunc(next lexStep) lexStep { + return func() (lexStep, error) { + for { + line, err := l.toEOL() + if err != nil { + return nil, err + } + + line = bytes.TrimSuffix(line, []byte{' '}) + + // lack of continuation means this line has been exhausted + if !bytes.HasSuffix(line, []byte{'\\'}) { + break + } + } + + // reached end of buffer, safe to exit + return next, nil + } +} + +func (l *lexer) lexNextSection() (lexStep, error) { + r, _, err := l.buf.ReadRune() + if err != nil { + if err == io.EOF { + err = nil + } + return nil, err + } + + if r == '[' { + return l.lexSectionName, nil + } else if isComment(r) { + return l.ignoreLineFunc(l.lexNextSection), nil + } + + return l.lexNextSection, nil +} + +func (l *lexer) lexNextSectionOrOptionFunc(section string) lexStep { + return func() (lexStep, error) { + r, _, err := l.buf.ReadRune() + if err != nil { + if err == io.EOF { + err = nil + } + return nil, err + } + + if unicode.IsSpace(r) { + return l.lexNextSectionOrOptionFunc(section), nil + } else if r == '[' { + return l.lexSectionName, nil + } else if isComment(r) { + return l.ignoreLineFunc(l.lexNextSectionOrOptionFunc(section)), nil + } + + l.buf.UnreadRune() + return l.lexOptionNameFunc(section), nil + } +} + +func (l *lexer) lexOptionNameFunc(section string) lexStep { + return func() (lexStep, error) { + var partial bytes.Buffer + for { + r, _, err := l.buf.ReadRune() + if err != nil { + return nil, err + } + + if r == '\n' || r == '\r' { + return nil, errors.New("unexpected newline encountered while parsing option name") + } + + if r == '=' { + break + } + + partial.WriteRune(r) + } + + name := strings.TrimSpace(partial.String()) + return l.lexOptionValueFunc(section, name), nil + } +} + +func (l *lexer) lexOptionValueFunc(section, name string) lexStep { + return func() (lexStep, error) { + var partial bytes.Buffer + + for { + line, err := l.toEOL() + if err != nil { + return nil, err + } + + // lack of continuation means this value has been exhausted + idx := bytes.LastIndex(line, []byte{'\\'}) + if idx == -1 || idx != (len(line)-1) { + partial.Write(line) + break + } + + partial.Write(line[0:idx]) + partial.WriteRune(' ') + } + + val := strings.TrimSpace(partial.String()) + l.optchan <- &UnitOption{Section: section, Name: name, Value: val} + + return l.lexNextSectionOrOptionFunc(section), nil + } +} + +func (l *lexer) toEOL() ([]byte, error) { + line, err := l.buf.ReadBytes('\n') + // ignore EOF here since it's roughly equivalent to EOL + if err != nil && err != io.EOF { + return nil, err + } + + line = bytes.TrimSuffix(line, []byte{'\r'}) + line = bytes.TrimSuffix(line, []byte{'\n'}) + + return line, nil +} + +func isComment(r rune) bool { + return r == '#' || r == ';' +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/deserialize_test.go b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/deserialize_test.go new file mode 100644 index 00000000000..bb98b82984e --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/deserialize_test.go @@ -0,0 +1,301 @@ +package unit + +import ( + "bytes" + "fmt" + "reflect" + "testing" +) + +func TestDeserialize(t *testing.T) { + tests := []struct { + input []byte + output []*UnitOption + }{ + // multiple options underneath a section + { + []byte(`[Unit] +Description=Foo +Description=Bar +Requires=baz.service +After=baz.service +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Foo"}, + &UnitOption{"Unit", "Description", "Bar"}, + &UnitOption{"Unit", "Requires", "baz.service"}, + &UnitOption{"Unit", "After", "baz.service"}, + }, + }, + + // multiple sections + { + []byte(`[Unit] +Description=Foo + +[Service] +ExecStart=/usr/bin/sleep infinity + +[X-Third-Party] +Pants=on + +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Foo"}, + &UnitOption{"Service", "ExecStart", "/usr/bin/sleep infinity"}, + &UnitOption{"X-Third-Party", "Pants", "on"}, + }, + }, + + // multiple sections with no options + { + []byte(`[Unit] +[Service] +[X-Third-Party] +`), + []*UnitOption{}, + }, + + // multiple values not special-cased + { + []byte(`[Service] +Environment= "FOO=BAR" "BAZ=QUX" +`), + []*UnitOption{ + &UnitOption{"Service", "Environment", "\"FOO=BAR\" \"BAZ=QUX\""}, + }, + }, + + // line continuations respected + { + []byte(`[Unit] +Description= Unnecessarily wrapped \ + words here +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Unnecessarily wrapped words here"}, + }, + }, + + // comments ignored + { + []byte(`; comment alpha +# comment bravo +[Unit] +; comment charlie +# comment delta +#Description=Foo +Description=Bar +; comment echo +# comment foxtrot +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Bar"}, + }, + }, + + // apparent comment lines inside of line continuations not ignored + { + []byte(`[Unit] +Description=Bar\ +# comment alpha + +Description=Bar\ +# comment bravo \ +Baz +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Bar # comment alpha"}, + &UnitOption{"Unit", "Description", "Bar # comment bravo Baz"}, + }, + }, + + // options outside of sections are ignored + { + []byte(`Description=Foo +[Unit] +Description=Bar +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Bar"}, + }, + }, + + // garbage outside of sections are ignored + { + []byte(`<<<<<<<< +[Unit] +Description=Bar +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Bar"}, + }, + }, + + // garbage used as unit option + { + []byte(`[Unit] +<<<<<<<<=Bar +`), + []*UnitOption{ + &UnitOption{"Unit", "<<<<<<<<", "Bar"}, + }, + }, + + // option name with spaces are valid + { + []byte(`[Unit] +Some Thing = Bar +`), + []*UnitOption{ + &UnitOption{"Unit", "Some Thing", "Bar"}, + }, + }, + + // lack of trailing newline doesn't cause problem for non-continued file + { + []byte(`[Unit] +Description=Bar`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Bar"}, + }, + }, + + // unit file with continuation but no following line is ok, too + { + []byte(`[Unit] +Description=Bar \`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "Bar"}, + }, + }, + + // Assert utf8 characters are preserved + { + []byte(`[©] +µ☃=ÇôrèÕ$`), + []*UnitOption{ + &UnitOption{"©", "µ☃", "ÇôrèÕ$"}, + }, + }, + + // whitespace removed around option name + { + []byte(`[Unit] + Description =words here +`), + []*UnitOption{ + &UnitOption{"Unit", "Description", "words here"}, + }, + }, + + // whitespace around option value stripped + { + []byte(`[Unit] +Description= words here `), + []*UnitOption{ + &UnitOption{"Unit", "Description", "words here"}, + }, + }, + + // whitespace around option value stripped, regardless of continuation + { + []byte(`[Unit] +Description= words here \ + `), + []*UnitOption{ + &UnitOption{"Unit", "Description", "words here"}, + }, + }, + + // backslash not considered continuation if followed by text + { + []byte(`[Service] +ExecStart=/bin/bash -c "while true; do echo \"ping\"; sleep 1; done" +`), + []*UnitOption{ + &UnitOption{"Service", "ExecStart", `/bin/bash -c "while true; do echo \"ping\"; sleep 1; done"`}, + }, + }, + + // backslash not considered continuation if followed by whitespace, but still trimmed + { + []byte(`[Service] +ExecStart=/bin/bash echo poof \ `), + []*UnitOption{ + &UnitOption{"Service", "ExecStart", `/bin/bash echo poof \`}, + }, + }, + } + + assert := func(expect, output []*UnitOption) error { + if len(expect) != len(output) { + return fmt.Errorf("expected %d items, got %d", len(expect), len(output)) + } + + for i, _ := range expect { + if !reflect.DeepEqual(expect[i], output[i]) { + return fmt.Errorf("item %d: expected %v, got %v", i, expect[i], output[i]) + } + } + + return nil + } + + for i, tt := range tests { + output, err := Deserialize(bytes.NewReader(tt.input)) + if err != nil { + t.Errorf("case %d: unexpected error parsing unit: %v", i, err) + continue + } + + err = assert(tt.output, output) + if err != nil { + t.Errorf("case %d: %v", i, err) + t.Log("Expected options:") + logUnitOptionSlice(t, tt.output) + t.Log("Actual options:") + logUnitOptionSlice(t, output) + } + } +} + +func TestDeserializeFail(t *testing.T) { + tests := [][]byte{ + // malformed section header + []byte(`[Unit +Description=Foo +`), + + // garbage following section header + []byte(`[Unit] pants +Description=Foo +`), + + // option without value + []byte(`[Unit] +Description +`), + + // garbage inside of section + []byte(`[Unit] +<<<<<< +Description=Foo +`), + } + + for i, tt := range tests { + output, err := Deserialize(bytes.NewReader(tt)) + if err == nil { + t.Errorf("case %d: unexpected non-nil error, received nil", i) + t.Log("Output:") + logUnitOptionSlice(t, output) + } + } +} + +func logUnitOptionSlice(t *testing.T, opts []*UnitOption) { + for idx, opt := range opts { + t.Logf("%d: %v", idx, opt) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/option.go b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/option.go new file mode 100644 index 00000000000..2b70fca36bc --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/option.go @@ -0,0 +1,36 @@ +package unit + +import ( + "fmt" +) + +type UnitOption struct { + Section string + Name string + Value string +} + +func (uo *UnitOption) String() string { + return fmt.Sprintf("{Section: %q, Name: %q, Value: %q}", uo.Section, uo.Name, uo.Value) +} + +func (uo *UnitOption) Match(other *UnitOption) bool { + return uo.Section == other.Section && + uo.Name == other.Name && + uo.Value == other.Value +} + +func AllMatch(u1 []*UnitOption, u2 []*UnitOption) bool { + length := len(u1) + if length != len(u2) { + return false + } + + for i := 0; i < length; i++ { + if !u1[i].Match(u2[i]) { + return false + } + } + + return true +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/option_test.go b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/option_test.go new file mode 100644 index 00000000000..ebdc062f797 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/option_test.go @@ -0,0 +1,200 @@ +package unit + +import ( + "testing" +) + +func TestAllMatch(t *testing.T) { + tests := []struct { + u1 []*UnitOption + u2 []*UnitOption + match bool + }{ + // empty lists match + { + u1: []*UnitOption{}, + u2: []*UnitOption{}, + match: true, + }, + + // simple match of a single option + { + u1: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + }, + u2: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + }, + match: true, + }, + + // single option mismatched + { + u1: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + }, + u2: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "BAR"}, + }, + match: false, + }, + + // multiple options match + { + u1: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + {Section: "Unit", Name: "BindsTo", Value: "bar.service"}, + {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, + }, + u2: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + {Section: "Unit", Name: "BindsTo", Value: "bar.service"}, + {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, + }, + match: true, + }, + + // mismatch length + { + u1: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + {Section: "Unit", Name: "BindsTo", Value: "bar.service"}, + }, + u2: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + {Section: "Unit", Name: "BindsTo", Value: "bar.service"}, + {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, + }, + match: false, + }, + + // multiple options misordered + { + u1: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, + }, + u2: []*UnitOption{ + {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, + {Section: "Unit", Name: "Description", Value: "FOO"}, + }, + match: false, + }, + + // interleaved sections mismatch + { + u1: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + {Section: "Unit", Name: "BindsTo", Value: "bar.service"}, + {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, + {Section: "Service", Name: "ExecStop", Value: "/bin/true"}, + }, + u2: []*UnitOption{ + {Section: "Unit", Name: "Description", Value: "FOO"}, + {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, + {Section: "Unit", Name: "BindsTo", Value: "bar.service"}, + {Section: "Service", Name: "ExecStop", Value: "/bin/true"}, + }, + match: false, + }, + } + + for i, tt := range tests { + match := AllMatch(tt.u1, tt.u2) + if match != tt.match { + t.Errorf("case %d: failed comparing u1 to u2 - expected match=%t, got %t", i, tt.match, match) + } + + match = AllMatch(tt.u2, tt.u1) + if match != tt.match { + t.Errorf("case %d: failed comparing u2 to u1 - expected match=%t, got %t", i, tt.match, match) + } + } +} + +func TestMatch(t *testing.T) { + tests := []struct { + o1 *UnitOption + o2 *UnitOption + match bool + }{ + // empty options match + { + o1: &UnitOption{}, + o2: &UnitOption{}, + match: true, + }, + + // all fields match + { + o1: &UnitOption{ + Section: "Unit", + Name: "Description", + Value: "FOO", + }, + o2: &UnitOption{ + Section: "Unit", + Name: "Description", + Value: "FOO", + }, + match: true, + }, + + // Section mismatch + { + o1: &UnitOption{ + Section: "Unit", + Name: "Description", + Value: "FOO", + }, + o2: &UnitOption{ + Section: "X-Other", + Name: "Description", + Value: "FOO", + }, + match: false, + }, + + // Name mismatch + { + o1: &UnitOption{ + Section: "Unit", + Name: "Description", + Value: "FOO", + }, + o2: &UnitOption{ + Section: "Unit", + Name: "BindsTo", + Value: "FOO", + }, + match: false, + }, + + // Value mismatch + { + o1: &UnitOption{ + Section: "Unit", + Name: "Description", + Value: "FOO", + }, + o2: &UnitOption{ + Section: "Unit", + Name: "Description", + Value: "BAR", + }, + match: false, + }, + } + + for i, tt := range tests { + match := tt.o1.Match(tt.o2) + if match != tt.match { + t.Errorf("case %d: failed comparing o1 to o2 - expected match=%t, got %t", i, tt.match, match) + } + + match = tt.o2.Match(tt.o1) + if match != tt.match { + t.Errorf("case %d: failed comparing o2 to o1 - expected match=%t, got %t", i, tt.match, match) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/serialize.go b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/serialize.go new file mode 100644 index 00000000000..867ce6000bb --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/serialize.go @@ -0,0 +1,51 @@ +package unit + +import ( + "bytes" + "io" +) + +// Serialize encodes all of the given UnitOption objects into a unit file +func Serialize(opts []*UnitOption) io.Reader { + var buf bytes.Buffer + + if len(opts) == 0 { + return &buf + } + + curSection := opts[0].Section + + writeSectionHeader(&buf, curSection) + writeNewline(&buf) + + for _, opt := range opts { + if opt.Section != curSection { + curSection = opt.Section + + writeNewline(&buf) + writeSectionHeader(&buf, curSection) + writeNewline(&buf) + } + + writeOption(&buf, opt) + writeNewline(&buf) + } + + return &buf +} + +func writeNewline(buf *bytes.Buffer) { + buf.WriteRune('\n') +} + +func writeSectionHeader(buf *bytes.Buffer, section string) { + buf.WriteRune('[') + buf.WriteString(section) + buf.WriteRune(']') +} + +func writeOption(buf *bytes.Buffer, opt *UnitOption) { + buf.WriteString(opt.Name) + buf.WriteRune('=') + buf.WriteString(opt.Value) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/serialize_test.go b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/serialize_test.go new file mode 100644 index 00000000000..e5ec0f13b7a --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-systemd/unit/serialize_test.go @@ -0,0 +1,134 @@ +package unit + +import ( + "io/ioutil" + "testing" +) + +func TestSerialize(t *testing.T) { + tests := []struct { + input []*UnitOption + output string + }{ + // no options results in empty file + { + []*UnitOption{}, + ``, + }, + + // options with same section share the header + { + []*UnitOption{ + &UnitOption{"Unit", "Description", "Foo"}, + &UnitOption{"Unit", "BindsTo", "bar.service"}, + }, + `[Unit] +Description=Foo +BindsTo=bar.service +`, + }, + + // options with same name are not combined + { + []*UnitOption{ + &UnitOption{"Unit", "Description", "Foo"}, + &UnitOption{"Unit", "Description", "Bar"}, + }, + `[Unit] +Description=Foo +Description=Bar +`, + }, + + // multiple options printed under different section headers + { + []*UnitOption{ + &UnitOption{"Unit", "Description", "Foo"}, + &UnitOption{"Service", "ExecStart", "/usr/bin/sleep infinity"}, + }, + `[Unit] +Description=Foo + +[Service] +ExecStart=/usr/bin/sleep infinity +`, + }, + + // no optimization for unsorted options + { + []*UnitOption{ + &UnitOption{"Unit", "Description", "Foo"}, + &UnitOption{"Service", "ExecStart", "/usr/bin/sleep infinity"}, + &UnitOption{"Unit", "BindsTo", "bar.service"}, + }, + `[Unit] +Description=Foo + +[Service] +ExecStart=/usr/bin/sleep infinity + +[Unit] +BindsTo=bar.service +`, + }, + + // utf8 characters are not a problem + { + []*UnitOption{ + &UnitOption{"©", "µ☃", "ÇôrèÕ$"}, + }, + `[©] +µ☃=ÇôrèÕ$ +`, + }, + + // no verification is done on section names + { + []*UnitOption{ + &UnitOption{"Un\nit", "Description", "Foo"}, + }, + `[Un +it] +Description=Foo +`, + }, + + // no verification is done on option names + { + []*UnitOption{ + &UnitOption{"Unit", "Desc\nription", "Foo"}, + }, + `[Unit] +Desc +ription=Foo +`, + }, + + // no verification is done on option values + { + []*UnitOption{ + &UnitOption{"Unit", "Description", "Fo\no"}, + }, + `[Unit] +Description=Fo +o +`, + }, + } + + for i, tt := range tests { + outReader := Serialize(tt.input) + outBytes, err := ioutil.ReadAll(outReader) + if err != nil { + t.Errorf("case %d: encountered error while reading output: %v", i, err) + continue + } + + output := string(outBytes) + if tt.output != output { + t.Errorf("case %d: incorrect output") + t.Logf("Expected:\n%s", tt.output) + t.Logf("Actual:\n%s", output) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/aci/aci.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/aci/aci.go new file mode 100644 index 00000000000..17cfff1da4e --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/aci/aci.go @@ -0,0 +1,170 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package aci implements helper functions for working with ACIs +package aci + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/appc/spec/aci" + "github.com/appc/spec/schema" + "golang.org/x/crypto/openpgp" +) + +type ACIEntry struct { + Header *tar.Header + Contents string +} + +type imageArchiveWriter struct { + *tar.Writer + am *schema.ImageManifest +} + +// NewImageWriter creates a new ArchiveWriter which will generate an App +// Container Image based on the given manifest and write it to the given +// tar.Writer +// TODO(sgotti) this is a copy of appc/spec/aci.imageArchiveWriter with +// addFileNow changed to create the file with the current user. needed for +// testing as non root user. +func NewImageWriter(am schema.ImageManifest, w *tar.Writer) aci.ArchiveWriter { + aw := &imageArchiveWriter{ + w, + &am, + } + return aw +} + +func (aw *imageArchiveWriter) AddFile(hdr *tar.Header, r io.Reader) error { + err := aw.Writer.WriteHeader(hdr) + if err != nil { + return err + } + + if r != nil { + _, err := io.Copy(aw.Writer, r) + if err != nil { + return err + } + } + + return nil +} + +func (aw *imageArchiveWriter) addFileNow(path string, contents []byte) error { + buf := bytes.NewBuffer(contents) + now := time.Now() + hdr := tar.Header{ + Name: path, + Mode: 0644, + Uid: os.Getuid(), + Gid: os.Getgid(), + Size: int64(buf.Len()), + ModTime: now, + Typeflag: tar.TypeReg, + ChangeTime: now, + } + return aw.AddFile(&hdr, buf) +} + +func (aw *imageArchiveWriter) addManifest(name string, m json.Marshaler) error { + out, err := m.MarshalJSON() + if err != nil { + return err + } + return aw.addFileNow(name, out) +} + +func (aw *imageArchiveWriter) Close() error { + if err := aw.addManifest(aci.ManifestFile, aw.am); err != nil { + return err + } + return aw.Writer.Close() +} + +// NewBasicACI creates a new ACI in the given directory with the given name. +// Used for testing. +func NewBasicACI(dir string, name string) (*os.File, error) { + manifest := fmt.Sprintf(`{"acKind":"ImageManifest","acVersion":"0.5.4","name":"%s"}`, name) + return NewACI(dir, manifest, nil) +} + +// NewACI creates a new ACI in the given directory with the given image +// manifest and entries. +// Used for testing. +func NewACI(dir string, manifest string, entries []*ACIEntry) (*os.File, error) { + var im schema.ImageManifest + if err := im.UnmarshalJSON([]byte(manifest)); err != nil { + return nil, err + } + + tf, err := ioutil.TempFile(dir, "") + if err != nil { + return nil, err + } + defer os.Remove(tf.Name()) + + tw := tar.NewWriter(tf) + aw := NewImageWriter(im, tw) + + for _, entry := range entries { + // Add default mode + if entry.Header.Mode == 0 { + if entry.Header.Typeflag == tar.TypeDir { + entry.Header.Mode = 0755 + } else { + entry.Header.Mode = 0644 + } + } + // Add calling user uid and gid or tests will fail + entry.Header.Uid = os.Getuid() + entry.Header.Gid = os.Getgid() + sr := strings.NewReader(entry.Contents) + if err := aw.AddFile(entry.Header, sr); err != nil { + return nil, err + } + } + + if err := aw.Close(); err != nil { + return nil, err + } + return tf, nil +} + +// NewDetachedSignature creates a new openpgp armored detached signature for the given ACI +// signed with armoredPrivateKey. +func NewDetachedSignature(armoredPrivateKey string, aci io.Reader) (io.Reader, error) { + entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(armoredPrivateKey)) + if err != nil { + return nil, err + } + if len(entityList) < 1 { + return nil, errors.New("empty entity list") + } + signature := &bytes.Buffer{} + if err := openpgp.ArmoredDetachSign(signature, entityList[0], aci, nil); err != nil { + return nil, err + } + return signature, nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/aci/render.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/aci/render.go new file mode 100644 index 00000000000..f7db562e35f --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/aci/render.go @@ -0,0 +1,61 @@ +package aci + +import ( + "archive/tar" + "fmt" + + "github.com/appc/spec/pkg/acirenderer" + "github.com/appc/spec/schema/types" + ptar "github.com/coreos/rkt/pkg/tar" +) + +// Given an imageID, start with the matching image available in the store, +// build its dependency list and render it inside dir +func RenderACIWithImageID(imageID types.Hash, dir string, ap acirenderer.ACIRegistry) error { + renderedACI, err := acirenderer.GetRenderedACIWithImageID(imageID, ap) + if err != nil { + return err + } + return renderImage(renderedACI, dir, ap) +} + +// Given an image app name and optional labels, get the best matching image +// available in the store, build its dependency list and render it inside dir +func RenderACI(name types.ACName, labels types.Labels, dir string, ap acirenderer.ACIRegistry) error { + renderedACI, err := acirenderer.GetRenderedACI(name, labels, ap) + if err != nil { + return err + } + return renderImage(renderedACI, dir, ap) +} + +// Given an already populated dependency list, it will extract, under the provided +// directory, the rendered ACI +func RenderACIFromList(imgs acirenderer.Images, dir string, ap acirenderer.ACIProvider) error { + renderedACI, err := acirenderer.GetRenderedACIFromList(imgs, ap) + if err != nil { + return err + } + return renderImage(renderedACI, dir, ap) +} + +// Given a RenderedACI, it will extract, under the provided directory, the +// needed files from the right source ACI. +// The manifest will be extracted from the upper ACI. +// No file overwriting is done as it should usually be called +// providing an empty directory. +func renderImage(renderedACI acirenderer.RenderedACI, dir string, ap acirenderer.ACIProvider) error { + for _, ra := range renderedACI { + rs, err := ap.ReadStream(ra.Key) + if err != nil { + return err + } + defer rs.Close() + // Overwrite is not needed. If a file needs to be overwritten then the renderedACI builder has a bug + if err := ptar.ExtractTar(tar.NewReader(rs), dir, false, ra.FileMap); err != nil { + return fmt.Errorf("error extracting ACI: %v", err) + } + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/dir.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/dir.go new file mode 100644 index 00000000000..265a2769242 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/dir.go @@ -0,0 +1,190 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package lock implements simple locking primitives on a +// regular file or directory using flock +package lock + +import ( + "errors" + "syscall" +) + +var ( + ErrLocked = errors.New("file already locked") + ErrNotExist = errors.New("file does not exist") + ErrPermission = errors.New("permission denied") + ErrNotRegular = errors.New("not a regular file") +) + +// FileLock represents a lock on a regular file or a directory +type FileLock struct { + path string + fd int +} + +type LockType int + +const ( + Dir LockType = iota + RegFile +) + +// TryExclusiveLock takes an exclusive lock without blocking. +// This is idempotent when the Lock already represents an exclusive lock, +// and tries promote a shared lock to exclusive atomically. +// It will return ErrLocked if any lock is already held. +func (l *FileLock) TryExclusiveLock() error { + err := syscall.Flock(l.fd, syscall.LOCK_EX|syscall.LOCK_NB) + if err == syscall.EWOULDBLOCK { + err = ErrLocked + } + return err +} + +// TryExclusiveLock takes an exclusive lock on a file/directory without blocking. +// It will return ErrLocked if any lock is already held on the file/directory. +func TryExclusiveLock(path string, lockType LockType) (*FileLock, error) { + l, err := NewLock(path, lockType) + if err != nil { + return nil, err + } + err = l.TryExclusiveLock() + if err != nil { + return nil, err + } + return l, err +} + +// ExclusiveLock takes an exclusive lock. +// This is idempotent when the Lock already represents an exclusive lock, +// and promotes a shared lock to exclusive atomically. +// It will block if an exclusive lock is already held. +func (l *FileLock) ExclusiveLock() error { + return syscall.Flock(l.fd, syscall.LOCK_EX) +} + +// ExclusiveLock takes an exclusive lock on a file/directory. +// It will block if an exclusive lock is already held on the file/directory. +func ExclusiveLock(path string, lockType LockType) (*FileLock, error) { + l, err := NewLock(path, lockType) + if err == nil { + err = l.ExclusiveLock() + } + if err != nil { + return nil, err + } + return l, nil +} + +// TrySharedLock takes a co-operative (shared) lock without blocking. +// This is idempotent when the Lock already represents a shared lock, +// and tries demote an exclusive lock to shared atomically. +// It will return ErrLocked if an exclusive lock already exists. +func (l *FileLock) TrySharedLock() error { + err := syscall.Flock(l.fd, syscall.LOCK_SH|syscall.LOCK_NB) + if err == syscall.EWOULDBLOCK { + err = ErrLocked + } + return err +} + +// TrySharedLock takes a co-operative (shared) lock on a file/directory without blocking. +// It will return ErrLocked if an exclusive lock already exists on the file/directory. +func TrySharedLock(path string, lockType LockType) (*FileLock, error) { + l, err := NewLock(path, lockType) + if err != nil { + return nil, err + } + err = l.TrySharedLock() + if err != nil { + return nil, err + } + return l, nil +} + +// SharedLock takes a co-operative (shared) lock on. +// This is idempotent when the Lock already represents a shared lock, +// and demotes an exclusive lock to shared atomically. +// It will block if an exclusive lock is already held. +func (l *FileLock) SharedLock() error { + return syscall.Flock(l.fd, syscall.LOCK_SH) +} + +// SharedLock takes a co-operative (shared) lock on a file/directory. +// It will block if an exclusive lock is already held on the file/directory. +func SharedLock(path string, lockType LockType) (*FileLock, error) { + l, err := NewLock(path, lockType) + if err != nil { + return nil, err + } + err = l.SharedLock() + if err != nil { + return nil, err + } + return l, nil +} + +// Unlock unlocks the lock +func (l *FileLock) Unlock() error { + return syscall.Flock(l.fd, syscall.LOCK_UN) +} + +// Fd returns the lock's file descriptor, or an error if the lock is closed +func (l *FileLock) Fd() (int, error) { + var err error + if l.fd == -1 { + err = errors.New("lock closed") + } + return l.fd, err +} + +// Close closes the lock which implicitly unlocks it as well +func (l *FileLock) Close() error { + fd := l.fd + l.fd = -1 + return syscall.Close(fd) +} + +// NewLock opens a new lock on a file without acquisition +func NewLock(path string, lockType LockType) (*FileLock, error) { + l := &FileLock{path: path, fd: -1} + + mode := syscall.O_RDONLY | syscall.O_CLOEXEC + if lockType == Dir { + mode |= syscall.O_DIRECTORY + } + lfd, err := syscall.Open(l.path, mode, 0) + if err != nil { + if err == syscall.ENOENT { + err = ErrNotExist + } else if err == syscall.EACCES { + err = ErrPermission + } + return nil, err + } + l.fd = lfd + + var stat syscall.Stat_t + err = syscall.Fstat(lfd, &stat) + if err != nil { + return nil, err + } + // Check if the file is a regular file + if lockType == RegFile && !(stat.Mode&syscall.S_IFMT == syscall.S_IFREG) { + return nil, ErrNotRegular + } + + return l, nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/dir_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/dir_test.go new file mode 100644 index 00000000000..fb866264a9a --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/dir_test.go @@ -0,0 +1,156 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lock + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestNewLock(t *testing.T) { + f, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("error creating tmpfile: %v", err) + } + defer os.Remove(f.Name()) + f.Close() + + l, err := NewLock(f.Name(), RegFile) + if err != nil { + t.Fatalf("error creating NewFileLock: %v", err) + } + l.Close() + + d, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.Remove(d) + + l, err = NewLock(d, Dir) + if err != nil { + t.Fatalf("error creating NewLock: %v", err) + } + + err = l.Close() + if err != nil { + t.Fatalf("error unlocking lock: %v", err) + } + + if err = os.Remove(d); err != nil { + t.Fatalf("error removing tmpdir: %v", err) + } + + l, err = NewLock(d, Dir) + if err == nil { + t.Fatalf("expected error creating lock on nonexistent path") + } +} + +func TestExclusiveLock(t *testing.T) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.Remove(dir) + + // Set up the initial exclusive lock + l, err := ExclusiveLock(dir, Dir) + if err != nil { + t.Fatalf("error creating lock: %v", err) + } + + // reacquire the exclusive lock using the receiver interface + err = l.TryExclusiveLock() + if err != nil { + t.Fatalf("error reacquiring exclusive lock: %v", err) + } + + // Now try another exclusive lock, should fail + _, err = TryExclusiveLock(dir, Dir) + if err == nil { + t.Fatalf("expected err trying exclusive lock") + } + + // Unlock the original lock + err = l.Close() + if err != nil { + t.Fatalf("error closing lock: %v", err) + } + + // Now another exclusive lock should succeed + _, err = TryExclusiveLock(dir, Dir) + if err != nil { + t.Fatalf("error creating lock: %v", err) + } +} + +func TestSharedLock(t *testing.T) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.Remove(dir) + + // Set up the initial shared lock + l1, err := SharedLock(dir, Dir) + if err != nil { + t.Fatalf("error creating new shared lock: %v", err) + } + + err = l1.TrySharedLock() + if err != nil { + t.Fatalf("error reacquiring shared lock: %v", err) + } + + // Subsequent shared locks should succeed + l2, err := TrySharedLock(dir, Dir) + if err != nil { + t.Fatalf("error creating shared lock: %v", err) + } + l3, err := TrySharedLock(dir, Dir) + if err != nil { + t.Fatalf("error creating shared lock: %v", err) + } + + // But an exclusive lock should fail + _, err = TryExclusiveLock(dir, Dir) + if err == nil { + t.Fatal("expected exclusive lock to fail") + } + + // Close the locks + err = l1.Close() + if err != nil { + t.Fatalf("error closing lock: %v", err) + } + err = l2.Close() + if err != nil { + t.Fatalf("error closing lock: %v", err) + } + + // Only unlock one of them + err = l3.Unlock() + if err != nil { + t.Fatalf("error unlocking lock: %v", err) + } + + // Now try an exclusive lock, should succeed + _, err = TryExclusiveLock(dir, Dir) + if err != nil { + t.Fatalf("error creating lock: %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/keylock.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/keylock.go new file mode 100644 index 00000000000..b91cda5923b --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/keylock.go @@ -0,0 +1,274 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lock + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +const ( + defaultDirPerm os.FileMode = 0660 + defaultFilePerm os.FileMode = 0660 + defaultLockRetries = 3 +) + +type keyLockMode uint + +const ( + keyLockExclusive keyLockMode = 1 << iota + keyLockShared + keyLockNonBlocking +) + +// KeyLock is a lock for a specific key. The lock file is created inside a +// directory using the key name. +// This is useful when multiple processes want to take a lock but cannot use +// FileLock as they don't have a well defined file on the filesystem. +// key value must be a valid file name (as the lock file is named after the key +// value). +type KeyLock struct { + lockDir string + key string + // The lock on the key + keyLock *FileLock +} + +// NewKeyLock returns a KeyLock for the specified key without acquisition. +// lockdir is the directory where the lock file will be created. If lockdir +// doesn't exists it will be created. +// key value must be a valid file name (as the lock file is named after the key +// value). +func NewKeyLock(lockDir string, key string) (*KeyLock, error) { + err := os.MkdirAll(lockDir, defaultDirPerm) + if err != nil { + return nil, err + } + keyLockFile := filepath.Join(lockDir, key) + // create the file if it doesn't exists + f, err := os.OpenFile(keyLockFile, os.O_RDONLY|os.O_CREATE, defaultFilePerm) + if err != nil { + return nil, fmt.Errorf("error creating key lock file: %v", err) + } + f.Close() + keyLock, err := NewLock(keyLockFile, RegFile) + if err != nil { + return nil, fmt.Errorf("error opening key lock file: %v", err) + } + return &KeyLock{lockDir: lockDir, key: key, keyLock: keyLock}, nil +} + +// Close closes the key lock which implicitly unlocks it as well +func (l *KeyLock) Close() { + l.keyLock.Close() +} + +// TryExclusiveLock takes an exclusive lock on a key without blocking. +// This is idempotent when the KeyLock already represents an exclusive lock, +// and tries promote a shared lock to exclusive atomically. +// It will return ErrLocked if any lock is already held on the key. +func (l *KeyLock) TryExclusiveKeyLock() error { + return l.lock(keyLockExclusive|keyLockNonBlocking, defaultLockRetries) +} + +// TryExclusiveLock takes an exclusive lock on the key without blocking. +// lockDir is the directory where the lock file will be created. +// It will return ErrLocked if any lock is already held. +func TryExclusiveKeyLock(lockDir string, key string) (*KeyLock, error) { + return createAndLock(lockDir, key, keyLockExclusive|keyLockNonBlocking) +} + +// ExclusiveLock takes an exclusive lock on a key. +// This is idempotent when the KeyLock already represents an exclusive lock, +// and promotes a shared lock to exclusive atomically. +// It will block if an exclusive lock is already held on the key. +func (l *KeyLock) ExclusiveKeyLock() error { + return l.lock(keyLockExclusive, defaultLockRetries) +} + +// ExclusiveLock takes an exclusive lock on a key. +// lockDir is the directory where the lock file will be created. +// It will block if an exclusive lock is already held on the key. +func ExclusiveKeyLock(lockDir string, key string) (*KeyLock, error) { + return createAndLock(lockDir, key, keyLockExclusive) +} + +// TrySharedLock takes a co-operative (shared) lock on the key without blocking. +// This is idempotent when the KeyLock already represents a shared lock, +// and tries demote an exclusive lock to shared atomically. +// It will return ErrLocked if an exclusive lock already exists on the key. +func (l *KeyLock) TrySharedKeyLock() error { + return l.lock(keyLockShared|keyLockNonBlocking, defaultLockRetries) +} + +// TrySharedLock takes a co-operative (shared) lock on a key without blocking. +// lockDir is the directory where the lock file will be created. +// It will return ErrLocked if an exclusive lock already exists on the key. +func TrySharedKeyLock(lockDir string, key string) (*KeyLock, error) { + return createAndLock(lockDir, key, keyLockShared|keyLockNonBlocking) +} + +// SharedLock takes a co-operative (shared) lock on a key. +// This is idempotent when the KeyLock already represents a shared lock, +// and demotes an exclusive lock to shared atomically. +// It will block if an exclusive lock is already held on the key. +func (l *KeyLock) SharedKeyLock() error { + return l.lock(keyLockShared, defaultLockRetries) +} + +// SharedLock takes a co-operative (shared) lock on a key. +// lockDir is the directory where the lock file will be created. +// It will block if an exclusive lock is already held on the key. +func SharedKeyLock(lockDir string, key string) (*KeyLock, error) { + return createAndLock(lockDir, key, keyLockShared) +} + +func createAndLock(lockDir string, key string, mode keyLockMode) (*KeyLock, error) { + keyLock, err := NewKeyLock(lockDir, key) + if err != nil { + return nil, err + } + err = keyLock.lock(mode, defaultLockRetries) + if err != nil { + keyLock.Close() + return nil, err + } + return keyLock, nil +} + +// lock is the base function to take a lock and handle changed lock files +// As there's the need to remove unused (see CleanKeyLocks) lock files without +// races, a changed file detection is needed. +// +// Without changed file detection this can happen: +// +// Process A takes exclusive lock on file01 +// Process B waits for exclusive lock on file01. +// Process A deletes file01 and then releases the lock. +// Process B takes the lock on the removed file01 as it has the fd opened +// Process C comes, creates the file as it doesn't exists, and it also takes an exclusive lock. +// Now B and C thinks to own an exclusive lock. +// +// maxRetries can be passed, useful for testing. +func (l *KeyLock) lock(mode keyLockMode, maxRetries int) error { + retries := 0 + for { + var err error + var isExclusive bool + var isNonBlocking bool + if mode&keyLockExclusive != 0 { + isExclusive = true + } + if mode&keyLockNonBlocking != 0 { + isNonBlocking = true + } + switch { + case isExclusive && !isNonBlocking: + err = l.keyLock.ExclusiveLock() + case isExclusive && isNonBlocking: + err = l.keyLock.TryExclusiveLock() + case !isExclusive && !isNonBlocking: + err = l.keyLock.SharedLock() + case !isExclusive && isNonBlocking: + err = l.keyLock.TrySharedLock() + } + if err != nil { + return err + } + + // Check that the file referenced by the lock fd is the same as + // the current file on the filesystem + var lockStat, curStat syscall.Stat_t + lfd, err := l.keyLock.Fd() + if err != nil { + return err + } + err = syscall.Fstat(lfd, &lockStat) + if err != nil { + return err + } + keyLockFile := filepath.Join(l.lockDir, l.key) + fd, err := syscall.Open(keyLockFile, syscall.O_RDONLY, 0) + // If there's an error opening the file return an error + if err != nil { + return err + } + if err := syscall.Fstat(fd, &curStat); err != nil { + syscall.Close(fd) + return err + } + syscall.Close(fd) + if lockStat.Ino == curStat.Ino && lockStat.Dev == curStat.Dev { + return nil + } + if retries >= maxRetries { + return fmt.Errorf("cannot acquire lock after %d retries", retries) + } + + // If the file has changed discard this lock and try to take another lock. + l.keyLock.Close() + nl, err := NewKeyLock(l.lockDir, l.key) + if err != nil { + return err + } + l.keyLock = nl.keyLock + + retries++ + } +} + +// Unlock unlocks the key lock. +func (l *KeyLock) Unlock() error { + err := l.keyLock.Unlock() + if err != nil { + return err + } + return nil +} + +// CleanKeyLocks remove lock files from the lockDir. +// For every key it tries to take an Exclusive lock on it and skip it if it +// fails with ErrLocked +func CleanKeyLocks(lockDir string) error { + f, err := os.Open(lockDir) + if err != nil { + return fmt.Errorf("error opening lockDir: %v", err) + } + defer f.Close() + files, err := f.Readdir(0) + if err != nil { + return fmt.Errorf("error getting lock files list: %v", err) + } + for _, f := range files { + filename := filepath.Join(lockDir, f.Name()) + keyLock, err := TryExclusiveKeyLock(lockDir, f.Name()) + if err == ErrLocked { + continue + } + if err != nil { + return err + } + + err = os.Remove(filename) + if err != nil { + keyLock.Close() + return fmt.Errorf("error removing lock file: %v", err) + } + keyLock.Close() + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/keylock_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/keylock_test.go new file mode 100644 index 00000000000..56cc9f1fd1d --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/lock/keylock_test.go @@ -0,0 +1,203 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lock + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestExclusiveKeyLock(t *testing.T) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.RemoveAll(dir) + + l1, err := ExclusiveKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating key lock: %v", err) + } + + _, err = TryExclusiveKeyLock(dir, "key01") + if err == nil { + t.Fatalf("expected err trying exclusive key lock") + } + + l1.Close() +} + +func TestCleanKeyLocks(t *testing.T) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.RemoveAll(dir) + + l1, err := ExclusiveKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + err = CleanKeyLocks(dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + filesnum, err := countFiles(dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if filesnum != 1 { + t.Fatalf("expected 1 file in lock dir. found %d files", filesnum) + } + + l2, err := SharedKeyLock(dir, "key02") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + l1.Close() + l2.Close() + + err = CleanKeyLocks(dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + filesnum, err = countFiles(dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if filesnum != 0 { + t.Fatalf("expected empty lock dir. found %d files", filesnum) + } +} + +func TestFileChangedLock(t *testing.T) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.RemoveAll(dir) + + l1, err := ExclusiveKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + l2, err := NewKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + // Simulate that l1 owner removes the actual key1 lock file + err = os.Remove(filepath.Join(dir, "key01")) + if err != nil { + t.Fatalf("error creating NewLock: %v", err) + } + l1.Close() + + // Now l2 owner takes a lock, using the fd of the removed file + err = l2.lock(keyLockShared, 0) + if err == nil { + t.Fatalf("expected error") + } + l2.Close() + + // Do the same with a new file created after removal + dir, err = ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.RemoveAll(dir) + + l1, err = ExclusiveKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + l2, err = NewKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + // Simulate that l1 owner removes the actual key1 lock file + err = os.Remove(filepath.Join(dir, "key01")) + if err != nil { + t.Fatalf("error creating NewLock: %v", err) + } + l1.Close() + + // Simulate that another user comes and takes a lock, this will create + // a new lock file as it was removed. + l3, err := ExclusiveKeyLock(dir, "key01") + l3.Close() + + // Now l2 owner takes a lock, using the fd of the old file + err = l2.lock(keyLockShared, 0) + if err == nil { + t.Fatalf("expected error") + } + + // Do the same but with a retry so if should work. + dir, err = ioutil.TempDir("", "") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + defer os.RemoveAll(dir) + + l1, err = ExclusiveKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + l2, err = NewKeyLock(dir, "key01") + if err != nil { + t.Fatalf("error creating keyLock: %v", err) + } + + // Simulate that l1 owner removes the actual key1 lock file + err = os.Remove(filepath.Join(dir, "key01")) + if err != nil { + t.Fatalf("error creating NewLock: %v", err) + } + l1.Close() + + // Simulate that another user comes and takes a lock, this will create + // a new lock file as it was removed. + l3, err = ExclusiveKeyLock(dir, "key01") + l3.Close() + + // Now l2 owner takes a lock, using the fd of the old file + err = l2.lock(keyLockShared, 1) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func countFiles(dir string) (int, error) { + f, err := os.Open(dir) + if err != nil { + return -1, err + } + defer f.Close() + files, err := f.Readdir(0) + if err != nil { + return -1, err + } + return len(files), nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys.go new file mode 100644 index 00000000000..0ba5d8fa80a --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys.go @@ -0,0 +1,18 @@ +package sys + +import ( + "syscall" +) + +// CloseOnExec sets or clears FD_CLOEXEC flag on a file descriptor +func CloseOnExec(fd int, set bool) error { + flag := uintptr(0) + if set { + flag = syscall.FD_CLOEXEC + } + _, _, err := syscall.RawSyscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFD, flag) + if err != 0 { + return syscall.Errno(err) + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux.go new file mode 100644 index 00000000000..eb67aca35bc --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux.go @@ -0,0 +1,11 @@ +package sys + +import "syscall" + +func Syncfs(fd int) error { + _, _, err := syscall.RawSyscall(SYS_SYNCFS, uintptr(fd), 0, 0) + if err != 0 { + return syscall.Errno(err) + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_386.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_386.go new file mode 100644 index 00000000000..3bdf0661122 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_386.go @@ -0,0 +1,5 @@ +package sys + +const ( + SYS_SYNCFS = 344 +) diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_amd64.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_amd64.go new file mode 100644 index 00000000000..d09054a9084 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_amd64.go @@ -0,0 +1,5 @@ +package sys + +const ( + SYS_SYNCFS = 306 +) diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_arm.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_arm.go new file mode 100644 index 00000000000..8ef6c0ec95c --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/sys/sys_linux_arm.go @@ -0,0 +1,5 @@ +package sys + +const ( + SYS_SYNCFS = 373 +) diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/tar.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/tar.go new file mode 100644 index 00000000000..333ab866871 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/tar.go @@ -0,0 +1,251 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tar contains helper functions for working with tar files +package tar + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "syscall" + "time" +) + +const DEFAULT_DIR_MODE os.FileMode = 0755 + +type insecureLinkError error + +var ErrNotSupportedPlatform = errors.New("platform and architecture is not supported") + +// Map of paths that should be whitelisted. The paths should be relative to the +// root of the tar file and should be cleaned (for example using filepath.Clean) +type PathWhitelistMap map[string]struct{} + +// ExtractTar extracts a tarball (from a tar.Reader) into the given directory +// if pwl is not nil, only the paths in the map are extracted. +// If overwrite is true, existing files will be overwritten. +func ExtractTar(tr *tar.Reader, dir string, overwrite bool, pwl PathWhitelistMap) error { + um := syscall.Umask(0) + defer syscall.Umask(um) + + dirhdrs := []*tar.Header{} +Tar: + for { + hdr, err := tr.Next() + switch err { + case io.EOF: + break Tar + case nil: + if pwl != nil { + relpath := filepath.Clean(hdr.Name) + if _, ok := pwl[relpath]; !ok { + continue + } + } + err = ExtractFile(tr, hdr, dir, overwrite) + if err != nil { + return fmt.Errorf("error extracting tarball: %v", err) + } + if hdr.Typeflag == tar.TypeDir { + dirhdrs = append(dirhdrs, hdr) + } + default: + return fmt.Errorf("error extracting tarball: %v", err) + } + } + + // Restore dirs atime and mtime. This has to be done after extracting + // as a file extraction will change its parent directory's times. + for _, hdr := range dirhdrs { + p := filepath.Join(dir, hdr.Name) + if err := syscall.UtimesNano(p, HdrToTimespec(hdr)); err != nil { + return err + } + } + return nil +} + +// ExtractFile extracts the file described by hdr from the given tarball into +// the provided directory. +// If overwrite is true, existing files will be overwritten. +func ExtractFile(tr *tar.Reader, hdr *tar.Header, dir string, overwrite bool) error { + p := filepath.Join(dir, hdr.Name) + fi := hdr.FileInfo() + typ := hdr.Typeflag + if overwrite { + info, err := os.Lstat(p) + switch { + case os.IsNotExist(err): + case err == nil: + // If the old and new paths are both dirs do nothing or + // RemoveAll will remove all dir's contents + if !info.IsDir() || typ != tar.TypeDir { + err := os.RemoveAll(p) + if err != nil { + return err + } + } + default: + return err + } + } + + // Create parent dir if it doesn't exist + if err := os.MkdirAll(filepath.Dir(p), DEFAULT_DIR_MODE); err != nil { + return err + } + switch { + case typ == tar.TypeReg || typ == tar.TypeRegA: + f, err := os.OpenFile(p, os.O_CREATE|os.O_RDWR, fi.Mode()) + if err != nil { + return err + } + _, err = io.Copy(f, tr) + if err != nil { + f.Close() + return err + } + f.Close() + case typ == tar.TypeDir: + if err := os.MkdirAll(p, fi.Mode()); err != nil { + return err + } + dir, err := os.Open(p) + if err != nil { + return err + } + if err := dir.Chmod(fi.Mode()); err != nil { + dir.Close() + return err + } + dir.Close() + case typ == tar.TypeLink: + dest := filepath.Join(dir, hdr.Linkname) + if !strings.HasPrefix(dest, dir) { + return insecureLinkError(fmt.Errorf("insecure link %q -> %q", p, hdr.Linkname)) + } + if err := os.Link(dest, p); err != nil { + return err + } + case typ == tar.TypeSymlink: + dest := filepath.Join(filepath.Dir(p), hdr.Linkname) + if !strings.HasPrefix(dest, dir) { + return insecureLinkError(fmt.Errorf("insecure symlink %q -> %q", p, hdr.Linkname)) + } + if err := os.Symlink(hdr.Linkname, p); err != nil { + return err + } + case typ == tar.TypeChar: + dev := makedev(int(hdr.Devmajor), int(hdr.Devminor)) + mode := uint32(fi.Mode()) | syscall.S_IFCHR + if err := syscall.Mknod(p, mode, dev); err != nil { + return err + } + case typ == tar.TypeBlock: + dev := makedev(int(hdr.Devmajor), int(hdr.Devminor)) + mode := uint32(fi.Mode()) | syscall.S_IFBLK + if err := syscall.Mknod(p, mode, dev); err != nil { + return err + } + case typ == tar.TypeFifo: + if err := syscall.Mkfifo(p, uint32(fi.Mode())); err != nil { + return err + } + // TODO(jonboulle): implement other modes + default: + return fmt.Errorf("unsupported type: %v", typ) + } + + if err := os.Lchown(p, hdr.Uid, hdr.Gid); err != nil { + return err + } + + // lchown(2) says that, depending on the linux kernel version, it + // can change the file's mode also if executed as root. So call + // os.Chmod after it. + if typ != tar.TypeSymlink { + if err := os.Chmod(p, fi.Mode()); err != nil { + return err + } + } + + // Restore entry atime and mtime. + // Use special function LUtimesNano not available on go's syscall package because we + // have to restore symlink's times and not the referenced file times. + ts := HdrToTimespec(hdr) + if hdr.Typeflag != tar.TypeSymlink { + if err := syscall.UtimesNano(p, ts); err != nil { + return err + } + } else { + if err := LUtimesNano(p, ts); err != nil && err != ErrNotSupportedPlatform { + return err + } + } + + return nil +} + +// ExtractFileFromTar extracts a regular file from the given tar, returning its +// contents as a byte slice +func ExtractFileFromTar(tr *tar.Reader, file string) ([]byte, error) { + for { + hdr, err := tr.Next() + switch err { + case io.EOF: + return nil, fmt.Errorf("file not found") + case nil: + if filepath.Clean(hdr.Name) != filepath.Clean(file) { + continue + } + switch hdr.Typeflag { + case tar.TypeReg: + case tar.TypeRegA: + default: + return nil, fmt.Errorf("requested file not a regular file") + } + buf, err := ioutil.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("error extracting tarball: %v", err) + } + return buf, nil + default: + return nil, fmt.Errorf("error extracting tarball: %v", err) + } + } +} + +// makedev mimics glib's gnu_dev_makedev +func makedev(major, minor int) int { + return (minor & 0xff) | (major & 0xfff << 8) | int((uint64(minor & ^0xff) << 12)) | int(uint64(major & ^0xfff)<<32) +} + +func HdrToTimespec(hdr *tar.Header) []syscall.Timespec { + return []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} +} + +// TODO(sgotti) use UTIMES_OMIT on linux if Time.IsZero ? +func timeToTimespec(time time.Time) (ts syscall.Timespec) { + nsec := int64(0) + if !time.IsZero() { + nsec = time.UnixNano() + } + return syscall.NsecToTimespec(nsec) +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/tar_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/tar_test.go new file mode 100644 index 00000000000..d97f51a4b81 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/tar_test.go @@ -0,0 +1,732 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tar + +import ( + "archive/tar" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +type testTarEntry struct { + header *tar.Header + contents string +} + +func newTestTar(entries []*testTarEntry) (string, error) { + t, err := ioutil.TempFile("", "test-tar") + if err != nil { + return "", err + } + defer t.Close() + tw := tar.NewWriter(t) + for _, entry := range entries { + // Add default mode + if entry.header.Mode == 0 { + if entry.header.Typeflag == tar.TypeDir { + entry.header.Mode = 0755 + } else { + entry.header.Mode = 0644 + } + } + // Add calling user uid and gid or tests will fail + entry.header.Uid = os.Getuid() + entry.header.Gid = os.Getgid() + if err := tw.WriteHeader(entry.header); err != nil { + return "", err + } + if _, err := io.WriteString(tw, entry.contents); err != nil { + return "", err + } + } + if err := tw.Close(); err != nil { + return "", err + } + return t.Name(), nil +} + +type fileInfo struct { + path string + typeflag byte + size int64 + contents string + mode os.FileMode +} + +func fileInfoSliceToMap(slice []*fileInfo) map[string]*fileInfo { + fim := make(map[string]*fileInfo, len(slice)) + for _, fi := range slice { + fim[fi.path] = fi + } + return fim +} + +func checkExpectedFiles(dir string, expectedFiles map[string]*fileInfo) error { + files := make(map[string]*fileInfo) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + fm := info.Mode() + if path == dir { + return nil + } + relpath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + switch { + case fm.IsRegular(): + files[relpath] = &fileInfo{path: relpath, typeflag: tar.TypeReg, size: info.Size(), mode: info.Mode().Perm()} + case info.IsDir(): + files[relpath] = &fileInfo{path: relpath, typeflag: tar.TypeDir, mode: info.Mode().Perm()} + case fm&os.ModeSymlink != 0: + files[relpath] = &fileInfo{path: relpath, typeflag: tar.TypeSymlink, mode: info.Mode()} + default: + return fmt.Errorf("file mode not handled: %v", fm) + } + + return nil + }) + if err != nil { + return err + } + + // Set defaults for not specified expected file mode + for _, ef := range expectedFiles { + if ef.mode == 0 { + if ef.typeflag == tar.TypeDir { + ef.mode = 0755 + } else { + ef.mode = 0644 + } + } + } + + for _, ef := range expectedFiles { + _, ok := files[ef.path] + if !ok { + return fmt.Errorf("Expected file %q not in files", ef.path) + } + + } + + for _, file := range files { + ef, ok := expectedFiles[file.path] + if !ok { + return fmt.Errorf("file %q not in expectedFiles", file.path) + } + if ef.typeflag != file.typeflag { + return fmt.Errorf("file %q: file type differs: wanted: %d, got: %d", file.path, ef.typeflag, file.typeflag) + } + if ef.typeflag == tar.TypeReg { + if ef.size != file.size { + return fmt.Errorf("file %q: size differs: wanted %d, wanted: %d", file.path, ef.size, file.size) + } + if ef.contents != "" { + buf, err := ioutil.ReadFile(filepath.Join(dir, file.path)) + if err != nil { + return fmt.Errorf("unexpected error: %v", err) + } + if string(buf) != ef.contents { + return fmt.Errorf("unexpected contents, wanted: %s, got: %s", ef.contents, buf) + } + + } + } + // Check modes but ignore symlinks + if ef.mode != file.mode && ef.typeflag != tar.TypeSymlink { + return fmt.Errorf("file %q: mode differs: wanted %#o, got: %#o", file.path, ef.mode, file.mode) + } + + } + return nil +} + +func TestExtractTarInsecureSymlink(t *testing.T) { + entries := []*testTarEntry{ + { + contents: "hello", + header: &tar.Header{ + Name: "hello.txt", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "link.txt", + Linkname: "hello.txt", + Typeflag: tar.TypeSymlink, + }, + }, + } + insecureSymlinkEntries := append(entries, &testTarEntry{ + header: &tar.Header{ + Name: "../etc/secret.conf", + Linkname: "secret.conf", + Typeflag: tar.TypeSymlink, + }, + }) + insecureHardlinkEntries := append(entries, &testTarEntry{ + header: &tar.Header{ + Name: "../etc/secret.conf", + Linkname: "secret.conf", + Typeflag: tar.TypeLink, + }, + }) + for _, entries := range [][]*testTarEntry{insecureSymlinkEntries, insecureHardlinkEntries} { + testTarPath, err := newTestTar(entries) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + defer os.Remove(testTarPath) + containerTar, err := os.Open(testTarPath) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + tr := tar.NewReader(containerTar) + tmpdir, err := ioutil.TempDir("", "rkt-temp-dir") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + os.RemoveAll(tmpdir) + err = os.MkdirAll(tmpdir, 0755) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer os.RemoveAll(tmpdir) + err = ExtractTar(tr, tmpdir, false, nil) + if _, ok := err.(insecureLinkError); !ok { + t.Errorf("expected insecureSymlinkError error") + } + } +} + +func TestExtractTarFolders(t *testing.T) { + entries := []*testTarEntry{ + { + contents: "foo", + header: &tar.Header{ + Name: "deep/folder/foo.txt", + Size: 3, + }, + }, + { + header: &tar.Header{ + Name: "deep/folder/", + Typeflag: tar.TypeDir, + Mode: int64(0747), + }, + }, + { + contents: "bar", + header: &tar.Header{ + Name: "deep/folder/bar.txt", + Size: 3, + }, + }, + { + header: &tar.Header{ + Name: "deep/folder2/symlink.txt", + Typeflag: tar.TypeSymlink, + Linkname: "deep/folder/foo.txt", + }, + }, + { + header: &tar.Header{ + Name: "deep/folder2/", + Typeflag: tar.TypeDir, + Mode: int64(0747), + }, + }, + { + contents: "bar", + header: &tar.Header{ + Name: "deep/folder2/bar.txt", + Size: 3, + }, + }, + { + header: &tar.Header{ + Name: "deep/deep/folder", + Typeflag: tar.TypeDir, + Mode: int64(0755), + }, + }, + { + header: &tar.Header{ + Name: "deep/deep/", + Typeflag: tar.TypeDir, + Mode: int64(0747), + }, + }, + } + + testTarPath, err := newTestTar(entries) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer os.Remove(testTarPath) + containerTar, err := os.Open(testTarPath) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + tr := tar.NewReader(containerTar) + tmpdir, err := ioutil.TempDir("", "rkt-temp-dir") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + os.RemoveAll(tmpdir) + err = os.MkdirAll(tmpdir, 0755) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer os.RemoveAll(tmpdir) + err = ExtractTar(tr, tmpdir, false, nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + matches, err := filepath.Glob(filepath.Join(tmpdir, "deep/folder/*.txt")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if len(matches) != 2 { + t.Errorf("unexpected number of files found: %d, wanted 2", len(matches)) + } + matches, err = filepath.Glob(filepath.Join(tmpdir, "deep/folder2/*.txt")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if len(matches) != 2 { + t.Errorf("unexpected number of files found: %d, wanted 2", len(matches)) + } + + dirInfo, err := os.Lstat(filepath.Join(tmpdir, "deep/folder")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } else if dirInfo.Mode().Perm() != os.FileMode(0747) { + t.Errorf("unexpected dir mode: %s", dirInfo.Mode()) + } + dirInfo, err = os.Lstat(filepath.Join(tmpdir, "deep/deep")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } else if dirInfo.Mode().Perm() != os.FileMode(0747) { + t.Errorf("unexpected dir mode: %s", dirInfo.Mode()) + } +} + +func TestExtractTarFileToBuf(t *testing.T) { + entries := []*testTarEntry{ + { + header: &tar.Header{ + Name: "folder/", + Typeflag: tar.TypeDir, + Mode: int64(0747), + }, + }, + { + contents: "foo", + header: &tar.Header{ + Name: "folder/foo.txt", + Size: 3, + }, + }, + { + contents: "bar", + header: &tar.Header{ + Name: "folder/bar.txt", + Size: 3, + }, + }, + { + header: &tar.Header{ + Name: "folder/symlink.txt", + Typeflag: tar.TypeSymlink, + Linkname: "folder/foo.txt", + }, + }, + } + testTarPath, err := newTestTar(entries) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer os.Remove(testTarPath) + containerTar, err := os.Open(testTarPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + tr := tar.NewReader(containerTar) + buf, err := ExtractFileFromTar(tr, "folder/foo.txt") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if string(buf) != "foo" { + t.Errorf("unexpected contents, wanted: %s, got: %s", "foo", buf) + } + containerTar.Close() + + containerTar, err = os.Open(testTarPath) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + tr = tar.NewReader(containerTar) + buf, err = ExtractFileFromTar(tr, "folder/symlink.txt") + if err == nil { + t.Errorf("expected error") + } + containerTar.Close() +} + +func TestExtractTarPWL(t *testing.T) { + entries := []*testTarEntry{ + { + header: &tar.Header{ + Name: "folder/", + Typeflag: tar.TypeDir, + Mode: int64(0747), + }, + }, + { + contents: "foo", + header: &tar.Header{ + Name: "folder/foo.txt", + Size: 3, + }, + }, + { + contents: "bar", + header: &tar.Header{ + Name: "folder/bar.txt", + Size: 3, + }, + }, + { + header: &tar.Header{ + Name: "folder/symlink.txt", + Typeflag: tar.TypeSymlink, + Linkname: "folder/foo.txt", + }, + }, + } + testTarPath, err := newTestTar(entries) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer os.Remove(testTarPath) + containerTar, err := os.Open(testTarPath) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + tr := tar.NewReader(containerTar) + tmpdir, err := ioutil.TempDir("", "rkt-temp-dir") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer os.RemoveAll(tmpdir) + + pwl := make(PathWhitelistMap) + pwl["folder/foo.txt"] = struct{}{} + err = ExtractTar(tr, tmpdir, false, pwl) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + matches, err := filepath.Glob(filepath.Join(tmpdir, "folder/*.txt")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if len(matches) != 1 { + t.Errorf("unexpected number of files found: %d, wanted 1", len(matches)) + } +} + +func TestExtractTarOverwrite(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "rkt-temp-dir") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer os.RemoveAll(tmpdir) + + entries := []*testTarEntry{ + { + contents: "hello", + header: &tar.Header{ + Name: "hello.txt", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "afolder", + Typeflag: tar.TypeDir, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "afolder/hello.txt", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "afile", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "folder01", + Typeflag: tar.TypeDir, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "folder01/file01", + Size: 5, + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "filesymlinked", + Size: 5, + }, + }, + { + header: &tar.Header{ + Name: "linktofile", + Linkname: "filesymlinked", + Typeflag: tar.TypeSymlink, + }, + }, + + { + header: &tar.Header{ + Name: "dirsymlinked", + Typeflag: tar.TypeDir, + }, + }, + { + header: &tar.Header{ + Name: "linktodir", + Linkname: "dirsymlinked", + Typeflag: tar.TypeSymlink, + }, + }, + } + + testTarPath, err := newTestTar(entries) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer os.Remove(testTarPath) + containerTar, err := os.Open(testTarPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tr := tar.NewReader(containerTar) + err = ExtractTar(tr, tmpdir, false, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Now overwrite: + // a file with a new file + // a dir with a file + entries = []*testTarEntry{ + { + contents: "newhello", + header: &tar.Header{ + Name: "hello.txt", + Size: 8, + }, + }, + // Now this is a file + { + contents: "nowafile", + header: &tar.Header{ + Name: "afolder", + Typeflag: tar.TypeReg, + Size: 8, + }, + }, + // Now this is a dir + { + header: &tar.Header{ + Name: "afile", + Typeflag: tar.TypeDir, + }, + }, + // Overwrite symlink to a file with a regular file + // the linked file shouldn't be removed + { + contents: "filereplacingsymlink", + header: &tar.Header{ + Name: "linktofile", + Typeflag: tar.TypeReg, + Size: 20, + }, + }, + // Overwrite symlink to a dir with a regular file + // the linked directory and all its contents shouldn't be + // removed + { + contents: "filereplacingsymlink", + header: &tar.Header{ + Name: "linktodir", + Typeflag: tar.TypeReg, + Size: 20, + }, + }, + // folder01 already exists and shouldn't be removed (keeping folder01/file01) + { + header: &tar.Header{ + Name: "folder01", + Typeflag: tar.TypeDir, + Mode: int64(0755), + }, + }, + { + contents: "hello", + header: &tar.Header{ + Name: "folder01/file02", + Size: 5, + Mode: int64(0644), + }, + }, + } + testTarPath, err = newTestTar(entries) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer os.Remove(testTarPath) + containerTar, err = os.Open(testTarPath) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + tr = tar.NewReader(containerTar) + err = ExtractTar(tr, tmpdir, true, nil) + + expectedFiles := []*fileInfo{ + &fileInfo{path: "hello.txt", typeflag: tar.TypeReg, size: 8, contents: "newhello"}, + &fileInfo{path: "linktofile", typeflag: tar.TypeReg, size: 20}, + &fileInfo{path: "linktodir", typeflag: tar.TypeReg, size: 20}, + &fileInfo{path: "afolder", typeflag: tar.TypeReg, size: 8}, + &fileInfo{path: "dirsymlinked", typeflag: tar.TypeDir}, + &fileInfo{path: "afile", typeflag: tar.TypeDir}, + &fileInfo{path: "filesymlinked", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "folder01", typeflag: tar.TypeDir}, + &fileInfo{path: "folder01/file01", typeflag: tar.TypeReg, size: 5}, + &fileInfo{path: "folder01/file02", typeflag: tar.TypeReg, size: 5}, + } + + err = checkExpectedFiles(tmpdir, fileInfoSliceToMap(expectedFiles)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestExtractTarTimes(t *testing.T) { + + // Do not set ns as tar has second precision + time1 := time.Unix(100000, 0) + time2 := time.Unix(200000, 0) + time3 := time.Unix(300000, 0) + entries := []*testTarEntry{ + { + header: &tar.Header{ + Name: "folder/", + Typeflag: tar.TypeDir, + Mode: int64(0747), + ModTime: time1, + }, + }, + { + contents: "foo", + header: &tar.Header{ + Name: "folder/foo.txt", + Size: 3, + ModTime: time2, + }, + }, + { + header: &tar.Header{ + Name: "folder/symlink.txt", + Typeflag: tar.TypeSymlink, + Linkname: "folder/foo.txt", + ModTime: time3, + }, + }, + } + + testTarPath, err := newTestTar(entries) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer os.Remove(testTarPath) + containerTar, err := os.Open(testTarPath) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + tr := tar.NewReader(containerTar) + tmpdir, err := ioutil.TempDir("", "rkt-temp-dir") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + os.RemoveAll(tmpdir) + err = os.MkdirAll(tmpdir, 0755) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + err = ExtractTar(tr, tmpdir, false, nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + err = checkTime(filepath.Join(tmpdir, "folder/"), time1) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + err = checkTime(filepath.Join(tmpdir, "folder/foo.txt"), time2) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + //Check only (by now) on linux + if runtime.GOOS == "linux" { + err = checkTime(filepath.Join(tmpdir, "folder/symlink.txt"), time3) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + } +} + +func checkTime(path string, time time.Time) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + + if info.ModTime() != time { + return fmt.Errorf("%s: info.ModTime: %s, different from expected time: %s", path, info.ModTime(), time) + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/utimes_linux.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/utimes_linux.go new file mode 100644 index 00000000000..77b89e9a908 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/utimes_linux.go @@ -0,0 +1,43 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// These functions are from github.com/docker/docker/pkg/system + +// TODO(sgotti) waiting for a utimensat functions accepting flags and a +// LUtimesNano using it in https://github.com/golang/sys/ + +package tar + +import ( + "syscall" + "unsafe" +) + +func LUtimesNano(path string, ts []syscall.Timespec) error { + // These are not currently available in syscall + AT_FDCWD := -100 + AT_SYMLINK_NOFOLLOW := 0x100 + + var _path *byte + _path, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + if _, _, err := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AT_FDCWD), uintptr(unsafe.Pointer(_path)), uintptr(unsafe.Pointer(&ts[0])), uintptr(AT_SYMLINK_NOFOLLOW), 0, 0); err != 0 && err != syscall.ENOSYS { + return err + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/utimes_unsupported.go b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/utimes_unsupported.go new file mode 100644 index 00000000000..02415812582 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/pkg/tar/utimes_unsupported.go @@ -0,0 +1,25 @@ +// +build !linux + +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// These functions are from github.com/docker/docker/pkg/system + +package tar + +import "syscall" + +func LUtimesNano(path string, ts []syscall.Timespec) error { + return ErrNotSupportedPlatform +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/aciinfo.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/aciinfo.go new file mode 100644 index 00000000000..be28b7c3f6f --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/aciinfo.go @@ -0,0 +1,87 @@ +package store + +import ( + "database/sql" + "time" +) + +// ACIInfo is used to store information about an ACI. +type ACIInfo struct { + // BlobKey is the key in the blob/imageManifest store of the related + // ACI file and is the db primary key. + BlobKey string + // AppName is the app name provided by the ACI. + AppName string + // ImportTime is the time this ACI was imported in the store. + ImportTime time.Time + // Latest defines if the ACI was imported using the latest pattern (no + // version label was provided on ACI discovery) + Latest bool +} + +func NewACIInfo(blobKey string, latest bool, t time.Time) *ACIInfo { + return &ACIInfo{ + BlobKey: blobKey, + Latest: latest, + ImportTime: t, + } +} + +// GetAciInfosWithKeyPrefix returns all the ACIInfos with a blobkey starting with the given prefix. +func GetACIInfosWithKeyPrefix(tx *sql.Tx, prefix string) ([]*ACIInfo, error) { + aciinfos := []*ACIInfo{} + rows, err := tx.Query("SELECT * from aciinfo WHERE hasPrefix(blobkey, $1)", prefix) + if err != nil { + return nil, err + } + for rows.Next() { + aciinfo := &ACIInfo{} + if err := rows.Scan(&aciinfo.BlobKey, &aciinfo.AppName, &aciinfo.ImportTime, &aciinfo.Latest); err != nil { + return nil, err + } + aciinfos = append(aciinfos, aciinfo) + } + if err := rows.Err(); err != nil { + return nil, err + } + return aciinfos, err +} + +// GetAciInfosWithAppName returns all the ACIInfos for a given appname. found will be +// false if no aciinfo exists. +func GetACIInfosWithAppName(tx *sql.Tx, appname string) ([]*ACIInfo, bool, error) { + aciinfos := []*ACIInfo{} + found := false + rows, err := tx.Query("SELECT * from aciinfo WHERE appname == $1", appname) + if err != nil { + return nil, false, err + } + for rows.Next() { + found = true + aciinfo := &ACIInfo{} + if err := rows.Scan(&aciinfo.BlobKey, &aciinfo.AppName, &aciinfo.ImportTime, &aciinfo.Latest); err != nil { + return nil, false, err + } + aciinfos = append(aciinfos, aciinfo) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + return aciinfos, found, err +} + +// WriteACIInfo adds or updates the provided aciinfo. +func WriteACIInfo(tx *sql.Tx, aciinfo *ACIInfo) error { + // ql doesn't have an INSERT OR UPDATE function so + // it's faster to remove and reinsert the row + _, err := tx.Exec("DELETE from aciinfo where blobkey == $1", aciinfo.BlobKey) + if err != nil { + return err + } + _, err = tx.Exec("INSERT into aciinfo values ($1, $2, $3, $4)", aciinfo.BlobKey, aciinfo.AppName, aciinfo.ImportTime, aciinfo.Latest) + if err != nil { + return err + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/aciinfo_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/aciinfo_test.go new file mode 100644 index 00000000000..46189dd6154 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/aciinfo_test.go @@ -0,0 +1,79 @@ +package store + +import ( + "database/sql" + "io/ioutil" + "os" + "testing" +) + +func TestWriteACIInfo(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if err = s.db.Do(func(tx *sql.Tx) error { + aciinfo := &ACIInfo{ + BlobKey: "key01", + AppName: "name01", + } + if err := WriteACIInfo(tx, aciinfo); err != nil { + return err + } + // Insert it another time to check that is should be overwritten + if err := WriteACIInfo(tx, aciinfo); err != nil { + return err + } + return nil + }); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + aciinfos := []*ACIInfo{} + ok := false + if err = s.db.Do(func(tx *sql.Tx) error { + aciinfos, ok, err = GetACIInfosWithAppName(tx, "name01") + return err + }); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !ok { + t.Fatalf("expected some records but none found") + } + if len(aciinfos) != 1 { + t.Fatalf("wrong number of records returned, wanted: 1, got: %d", len(aciinfos)) + } + + // Add another ACIInfo for the same app name + if err = s.db.Do(func(tx *sql.Tx) error { + aciinfo := &ACIInfo{ + BlobKey: "key02", + AppName: "name01", + } + if err := WriteACIInfo(tx, aciinfo); err != nil { + return err + } + return nil + }); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err = s.db.Do(func(tx *sql.Tx) error { + aciinfos, ok, err = GetACIInfosWithAppName(tx, "name01") + return err + }); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !ok { + t.Fatalf("expected some records but none found") + } + if len(aciinfos) != 2 { + t.Fatalf("wrong number of records returned, wanted: 2, got: %d", len(aciinfos)) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/db.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/db.go new file mode 100644 index 00000000000..56392c40aa1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/db.go @@ -0,0 +1,114 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + + "github.com/coreos/rkt/pkg/lock" + + _ "github.com/cznic/ql/driver" +) + +const ( + DbFilename = "ql.db" +) + +type DB struct { + dbdir string + lock *lock.FileLock + sqldb *sql.DB +} + +func NewDB(dbdir string) (*DB, error) { + if err := os.MkdirAll(dbdir, defaultPathPerm); err != nil { + return nil, err + } + return &DB{dbdir: dbdir}, nil +} + +func (db *DB) Open() error { + // take a lock on db dir + if db.lock != nil { + panic("cas db lock already gained") + } + dl, err := lock.ExclusiveLock(db.dbdir, lock.Dir) + if err != nil { + return err + } + db.lock = dl + + sqldb, err := sql.Open("ql", filepath.Join(db.dbdir, DbFilename)) + if err != nil { + dl.Close() + return err + } + db.sqldb = sqldb + return nil +} + +func (db *DB) Close() error { + if db.lock == nil { + panic("cas db, Close called without lock") + } + if db.sqldb == nil { + panic("cas db, Close called without an open sqldb") + } + if err := db.sqldb.Close(); err != nil { + return fmt.Errorf("cas db close failed: %v", err) + } + db.sqldb = nil + + db.lock.Close() + db.lock = nil + return nil +} + +func (db *DB) Begin() (*sql.Tx, error) { + return db.sqldb.Begin() +} + +type txfunc func(*sql.Tx) error + +// Do Opens the db, executes DoTx and then Closes the DB +func (db *DB) Do(fns ...txfunc) error { + err := db.Open() + if err != nil { + return err + } + defer db.Close() + + return db.DoTx(fns...) +} + +// DoTx executes the provided txfuncs inside a unique transaction. +// If one of the functions returns an error the whole transaction is rolled back. +func (db *DB) DoTx(fns ...txfunc) error { + tx, err := db.Begin() + if err != nil { + return err + } + for _, fn := range fns { + if err := fn(tx); err != nil { + tx.Rollback() + return err + } + } + tx.Commit() + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/migrate.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/migrate.go new file mode 100644 index 00000000000..5a906210a07 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/migrate.go @@ -0,0 +1,59 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "database/sql" + "fmt" +) + +type migrateFunc func(*sql.Tx) error + +var ( + // migrateTable is a map of migrate functions. The key is the db + // version to migrate to. + migrateTable = map[int]migrateFunc{ + 1: migrateToV1, + } +) + +func migrate(tx *sql.Tx, finalVersion int) error { + if finalVersion > dbVersion { + return fmt.Errorf("required migrate final version greater than the last supported db version") + } + version, err := getDBVersion(tx) + if err != nil { + return err + } + + for v := version + 1; v <= finalVersion; v++ { + f, ok := migrateTable[v] + if !ok { + return fmt.Errorf("missing migrate function for version %d", v) + } + err := f(tx) + if err == nil { + updateDBVersion(tx, v) + } + if err != nil { + return fmt.Errorf("failed to migrate db to version %d: %v", v, err) + } + } + return nil +} + +func migrateToV1(tx *sql.Tx) error { + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/migrate_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/migrate_test.go new file mode 100644 index 00000000000..8a5a241bb8e --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/migrate_test.go @@ -0,0 +1,396 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "database/sql" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +type testdb interface { + version() int + populate(db *DB) error + load(db *DB) error + compare(db testdb) bool +} + +type DBV0 struct { + aciinfos []*ACIInfoV0_1 + remotes []*RemoteV0_1 +} + +func (d *DBV0) version() int { + return 0 +} + +func (d *DBV0) populate(db *DB) error { + // As DBV0 and DBV1 have the same schema use a common populate + // function. + return populateDBV0_1(db, d.version(), d.aciinfos, d.remotes) +} + +// load populates the given struct with the data in db. +// the given struct d should be empty +func (d *DBV0) load(db *DB) error { + fn := func(tx *sql.Tx) error { + var err error + d.aciinfos, err = getAllACIInfosV0_1(tx) + if err != nil { + return err + } + d.remotes, err = getAllRemoteV0_1(tx) + if err != nil { + return err + } + return nil + } + if err := db.Do(fn); err != nil { + return err + } + return nil +} + +func (d *DBV0) compare(td testdb) bool { + d2, ok := td.(*DBV0) + if !ok { + return false + } + if !compareSlicesNoOrder(d.aciinfos, d2.aciinfos) { + return false + } + if !compareSlicesNoOrder(d.remotes, d2.remotes) { + return false + } + return true +} + +type DBV1 struct { + aciinfos []*ACIInfoV0_1 + remotes []*RemoteV0_1 +} + +func (d *DBV1) version() int { + return 1 +} +func (d *DBV1) populate(db *DB) error { + return populateDBV0_1(db, d.version(), d.aciinfos, d.remotes) +} + +func (d *DBV1) load(db *DB) error { + fn := func(tx *sql.Tx) error { + var err error + d.aciinfos, err = getAllACIInfosV0_1(tx) + if err != nil { + return err + } + d.remotes, err = getAllRemoteV0_1(tx) + if err != nil { + return err + } + return nil + } + if err := db.Do(fn); err != nil { + return err + } + return nil +} + +func (d *DBV1) compare(td testdb) bool { + d2, ok := td.(*DBV1) + if !ok { + return false + } + if !compareSlicesNoOrder(d.aciinfos, d2.aciinfos) { + return false + } + if !compareSlicesNoOrder(d.remotes, d2.remotes) { + return false + } + return true +} + +// The ACIInfo struct for different db versions. The ending VX_Y represent the +// first and the last version where the format isn't changed +// The latest existing struct should to be updated when updating the db version +// without changing the struct format (ex. V0_1 to V0_2). +// A new struct and its relative function should be added if the format is changed. +// The same applies for all of the the other structs. +type ACIInfoV0_1 struct { + BlobKey string + AppName string + ImportTime time.Time + Latest bool +} + +func getAllACIInfosV0_1(tx *sql.Tx) ([]*ACIInfoV0_1, error) { + aciinfos := []*ACIInfoV0_1{} + rows, err := tx.Query("SELECT * from aciinfo") + if err != nil { + return nil, err + } + for rows.Next() { + aciinfo := &ACIInfoV0_1{} + if err := rows.Scan(&aciinfo.BlobKey, &aciinfo.AppName, &aciinfo.ImportTime, &aciinfo.Latest); err != nil { + return nil, err + } + aciinfos = append(aciinfos, aciinfo) + } + if err := rows.Err(); err != nil { + return nil, err + } + return aciinfos, nil +} + +type RemoteV0_1 struct { + ACIURL string + SigURL string + ETag string + BlobKey string +} + +func getAllRemoteV0_1(tx *sql.Tx) ([]*RemoteV0_1, error) { + remotes := []*RemoteV0_1{} + rows, err := tx.Query("SELECT * from remote") + if err != nil { + return nil, err + } + for rows.Next() { + remote := &RemoteV0_1{} + if err := rows.Scan(&remote.ACIURL, &remote.SigURL, &remote.ETag, &remote.BlobKey); err != nil { + return nil, err + } + remotes = append(remotes, remote) + } + if err := rows.Err(); err != nil { + return nil, err + } + return remotes, nil +} + +func populateDBV0_1(db *DB, dbVersion int, aciInfos []*ACIInfoV0_1, remotes []*RemoteV0_1) error { + var dbCreateStmts = [...]string{ + // version table + "CREATE TABLE IF NOT EXISTS version (version int);", + fmt.Sprintf("INSERT INTO version VALUES (%d)", dbVersion), + + // remote table. The primary key is "aciurl". + "CREATE TABLE IF NOT EXISTS remote (aciurl string, sigurl string, etag string, blobkey string);", + "CREATE UNIQUE INDEX IF NOT EXISTS aciurlidx ON remote (aciurl)", + + // aciinfo table. The primary key is "blobkey" and it matches the key used to save that aci in the blob store + "CREATE TABLE IF NOT EXISTS aciinfo (blobkey string, appname string, importtime time, latest bool);", + "CREATE UNIQUE INDEX IF NOT EXISTS blobkeyidx ON aciinfo (blobkey)", + "CREATE INDEX IF NOT EXISTS appnameidx ON aciinfo (appname)", + } + + fn := func(tx *sql.Tx) error { + for _, stmt := range dbCreateStmts { + _, err := tx.Exec(stmt) + if err != nil { + return err + } + } + return nil + } + if err := db.Do(fn); err != nil { + return err + } + + fn = func(tx *sql.Tx) error { + for _, aciinfo := range aciInfos { + _, err := tx.Exec("INSERT into aciinfo values ($1, $2, $3, $4)", aciinfo.BlobKey, aciinfo.AppName, aciinfo.ImportTime, aciinfo.Latest) + if err != nil { + return err + } + } + return nil + } + if err := db.Do(fn); err != nil { + return err + } + + fn = func(tx *sql.Tx) error { + for _, remote := range remotes { + _, err := tx.Exec("INSERT into remote values ($1, $2, $3, $4)", remote.ACIURL, remote.SigURL, remote.ETag, remote.BlobKey) + if err != nil { + return err + } + } + return nil + } + if err := db.Do(fn); err != nil { + return err + } + + return nil +} + +type migrateTest struct { + predb testdb + postdb testdb + // Needed to have the right DB type to load from + curdb testdb +} + +func testMigrate(tt migrateTest) error { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + return fmt.Errorf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + + casDir := filepath.Join(dir, "cas") + db, err := NewDB(filepath.Join(casDir, "db")) + if err != nil { + return err + } + if err = tt.predb.populate(db); err != nil { + return err + } + + fn := func(tx *sql.Tx) error { + err := migrate(tx, tt.postdb.version()) + if err != nil { + return err + } + return nil + } + if err = db.Do(fn); err != nil { + return err + } + + var curDBVersion int + fn = func(tx *sql.Tx) error { + var err error + curDBVersion, err = getDBVersion(tx) + if err != nil { + return err + } + return nil + } + if err = db.Do(fn); err != nil { + return err + } + if curDBVersion != tt.postdb.version() { + return fmt.Errorf("wrong db version: got %#v, want %#v", curDBVersion, tt.postdb.version()) + } + + if err := tt.curdb.load(db); err != nil { + return err + } + if !tt.curdb.compare(tt.postdb) { + // TODO(sgotti) not very useful as these are pointers. + // Use something like go-spew to write the full data? + return fmt.Errorf("got %#v, want %#v", tt.curdb, tt.postdb) + } + return nil +} + +func TestMigrate(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + + now := time.Now() + tests := []migrateTest{ + // Test migration from V0 to V1 + // Empty db + { + &DBV0{}, + &DBV1{}, + &DBV1{}, + }, + { + &DBV0{ + []*ACIInfoV0_1{ + {"sha512-aaaaaaaa", "example.com/app01", now, false}, + {"sha512-bbbbbbbb", "example.com/app02", now, true}, + }, + []*RemoteV0_1{ + {"http://example.com/app01.aci", "http://example.com/app01.aci.asc", "", "sha512-aaaaaaaa"}, + {"http://example.com/app02.aci", "http://example.com/app02.aci.asc", "", "sha512-bbbbbbbb"}, + }, + }, + &DBV1{ + []*ACIInfoV0_1{ + {"sha512-aaaaaaaa", "example.com/app01", now, false}, + {"sha512-bbbbbbbb", "example.com/app02", now, true}, + }, + []*RemoteV0_1{ + {"http://example.com/app01.aci", "http://example.com/app01.aci.asc", "", "sha512-aaaaaaaa"}, + {"http://example.com/app02.aci", "http://example.com/app02.aci.asc", "", "sha512-bbbbbbbb"}, + }, + }, + &DBV1{}, + }, + } + + for i, tt := range tests { + if err := testMigrate(tt); err != nil { + t.Errorf("#%d: unexpected error: %v", i, err) + } + } +} + +// compareSlices compare slices regardless of the slice elements order +func compareSlicesNoOrder(i1 interface{}, i2 interface{}) bool { + s1 := interfaceToSlice(i1) + s2 := interfaceToSlice(i2) + + if len(s1) != len(s2) { + return false + } + + seen := map[int]bool{} + for _, v1 := range s1 { + found := false + for i2, v2 := range s2 { + if _, ok := seen[i2]; ok { + continue + } + if reflect.DeepEqual(v1, v2) { + found = true + seen[i2] = true + continue + } + } + if !found { + return false + } + + } + return true +} + +func interfaceToSlice(s interface{}) []interface{} { + v := reflect.ValueOf(s) + if v.Kind() != reflect.Slice && v.Kind() != reflect.Array { + panic(fmt.Errorf("Expected slice or array, got %T", s)) + } + l := v.Len() + m := make([]interface{}, l) + for i := 0; i < l; i++ { + m[i] = v.Index(i).Interface() + } + return m +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/remote.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/remote.go new file mode 100644 index 00000000000..43458898921 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/remote.go @@ -0,0 +1,72 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package store implements a content-addressable-store on disk. +// It leverages the `diskv` package to store items in a simple +// key-value blob store: https://github.com/peterbourgon/diskv +package store + +import "database/sql" + +func NewRemote(aciurl, sigurl string) *Remote { + r := &Remote{ + ACIURL: aciurl, + SigURL: sigurl, + } + return r +} + +type Remote struct { + ACIURL string + SigURL string + ETag string + // The key in the blob store under which the ACI has been saved. + BlobKey string +} + +// GetRemote tries to retrieve a remote with the given aciURL. found will be +// false if remote doesn't exist. +func GetRemote(tx *sql.Tx, aciURL string) (remote *Remote, found bool, err error) { + remote = &Remote{} + rows, err := tx.Query("SELECT sigurl, etag, blobkey FROM remote WHERE aciurl == $1", aciURL) + if err != nil { + return nil, false, err + } + for rows.Next() { + found = true + if err := rows.Scan(&remote.SigURL, &remote.ETag, &remote.BlobKey); err != nil { + return nil, false, err + } + } + if err := rows.Err(); err != nil { + return nil, false, err + } + + return remote, found, err +} + +// WriteRemote adds or updates the provided Remote. +func WriteRemote(tx *sql.Tx, remote *Remote) error { + // ql doesn't have an INSERT OR UPDATE function so + // it's faster to remove and reinsert the row + _, err := tx.Exec("DELETE FROM remote WHERE aciurl == $1", remote.ACIURL) + if err != nil { + return err + } + _, err = tx.Exec("INSERT INTO remote VALUES ($1, $2, $3, $4)", remote.ACIURL, remote.SigURL, remote.ETag, remote.BlobKey) + if err != nil { + return err + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/remote_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/remote_test.go new file mode 100644 index 00000000000..bf23abe9a18 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/remote_test.go @@ -0,0 +1,66 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestNewRemote(t *testing.T) { + const ( + u1 = "https://example.com" + u2 = "https://foo.com" + data = "asdf" + ) + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatal(err) + } + + // Create our first Remote, and simulate Store() to create row in the table + na := NewRemote(u1, "") + na.BlobKey = data + s.WriteRemote(na) + + // Get a new remote w the same parameters, reading from table should be fine + nb, ok, err := s.GetRemote(u1) + if err != nil { + t.Fatalf("unexpected error reading index: %v", err) + } + if !ok { + t.Fatalf("unexpected index not found") + } + if nb.BlobKey != data { + t.Fatalf("bad data returned from store: got %v, want %v", nb.BlobKey, data) + } + + // Get a remote with a different URI + nc, ok, err := s.GetRemote(u2) + // Should get an error, since the URI shouldn't be present in the table + if ok { + t.Fatalf("unexpected index found") + } + // Remote shouldn't be populated + if nc.BlobKey != "" { + t.Errorf("unexpected blob: got %v", nc.BlobKey) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/schema.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/schema.go new file mode 100644 index 00000000000..a2d3bf75335 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/schema.go @@ -0,0 +1,87 @@ +package store + +import ( + "database/sql" + "fmt" +) + +const ( + // Incremental db version at the current code revision. + dbVersion = 1 +) + +// Statement to run when creating a db. These are the statements to create the +// db at the latest db version (dbVersion) provided by this rkt version. +// If the db already exists migration statements should be executed +var dbCreateStmts = [...]string{ + // version table + "CREATE TABLE IF NOT EXISTS version (version int);", + fmt.Sprintf("INSERT INTO version VALUES (%d)", dbVersion), + + // remote table. The primary key is "aciurl". + "CREATE TABLE IF NOT EXISTS remote (aciurl string, sigurl string, etag string, blobkey string);", + "CREATE UNIQUE INDEX IF NOT EXISTS aciurlidx ON remote (aciurl)", + + // aciinfo table. The primary key is "blobkey" and it matches the key used to save that aci in the blob store + "CREATE TABLE IF NOT EXISTS aciinfo (blobkey string, appname string, importtime time, latest bool);", + "CREATE UNIQUE INDEX IF NOT EXISTS blobkeyidx ON aciinfo (blobkey)", + "CREATE INDEX IF NOT EXISTS appnameidx ON aciinfo (appname)", +} + +// dbIsPopulated checks if the db is already populated (at any version) verifing if the "version" table exists +func dbIsPopulated(tx *sql.Tx) (bool, error) { + rows, err := tx.Query("SELECT Name FROM __Table where Name == $1", "version") + if err != nil { + return false, err + } + count := 0 + for rows.Next() { + count++ + } + if err := rows.Err(); err != nil { + return false, err + } + if count > 0 { + return true, nil + } + return false, nil +} + +// getDBVersion retrieves the current db version +func getDBVersion(tx *sql.Tx) (int, error) { + var version int + rows, err := tx.Query("SELECT version FROM version") + if err != nil { + return -1, err + } + found := false + for rows.Next() { + if err := rows.Scan(&version); err != nil { + return -1, err + } + found = true + break + } + if err := rows.Err(); err != nil { + return -1, err + } + if !found { + return -1, fmt.Errorf("db version table empty") + } + return version, nil +} + +// updateDBVersion updates the db version +func updateDBVersion(tx *sql.Tx, version int) error { + // ql doesn't have an INSERT OR UPDATE function so + // it's faster to remove and reinsert the row + _, err := tx.Exec("DELETE FROM version") + if err != nil { + return err + } + _, err = tx.Exec("INSERT INTO version VALUES ($1)", version) + if err != nil { + return err + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/store.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/store.go new file mode 100644 index 00000000000..17246e27012 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/store.go @@ -0,0 +1,547 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "bufio" + "bytes" + "crypto/sha512" + "database/sql" + "encoding/json" + "fmt" + "hash" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/coreos/rkt/pkg/lock" + + "github.com/appc/spec/aci" + "github.com/appc/spec/schema" + "github.com/appc/spec/schema/types" + + "github.com/peterbourgon/diskv" +) + +const ( + blobType int64 = iota + imageManifestType + + defaultPathPerm os.FileMode = 0777 + defaultFilePerm os.FileMode = 0660 + + // To ameliorate excessively long paths, keys for the (blob)store use + // only the first half of a sha512 rather than the entire sum + hashPrefix = "sha512-" + lenHash = sha512.Size // raw byte size + lenHashKey = (lenHash / 2) * 2 // half length, in hex characters + lenKey = len(hashPrefix) + lenHashKey + minlenKey = len(hashPrefix) + 2 // at least sha512-aa +) + +var diskvStores = [...]string{ + "blob", + "imageManifest", +} + +// Store encapsulates a content-addressable-storage for storing ACIs on disk. +type Store struct { + base string + stores []*diskv.Diskv + db *DB + treestore *TreeStore + // storeLock is a lock on the whole store. It's used for store migration. If + // a previous version of rkt is using the store and in the meantime a + // new version is installed and executed it will try migrate the store + // during NewStore. This means that the previous running rkt will fail + // or behave badly after the migration as it's expecting another db format. + // For this reason, before executing migration, an exclusive lock must + // be taken on the whole store. + storeLock *lock.FileLock + imageLockDir string + treeStoreLockDir string +} + +func NewStore(base string) (*Store, error) { + casDir := filepath.Join(base, "cas") + + s := &Store{ + base: base, + stores: make([]*diskv.Diskv, len(diskvStores)), + } + + s.imageLockDir = filepath.Join(casDir, "imagelocks") + err := os.MkdirAll(s.imageLockDir, defaultPathPerm) + if err != nil { + return nil, err + } + + s.treeStoreLockDir = filepath.Join(casDir, "treestorelocks") + err = os.MkdirAll(s.treeStoreLockDir, defaultPathPerm) + if err != nil { + return nil, err + } + + // Take a shared cas lock + s.storeLock, err = lock.NewLock(casDir, lock.Dir) + if err != nil { + return nil, err + } + + for i, p := range diskvStores { + s.stores[i] = diskv.New(diskv.Options{ + BasePath: filepath.Join(casDir, p), + Transform: blockTransform, + }) + } + db, err := NewDB(filepath.Join(casDir, "db")) + if err != nil { + return nil, err + } + s.db = db + + s.treestore = &TreeStore{path: filepath.Join(base, "cas", "tree")} + + needsMigrate := false + fn := func(tx *sql.Tx) error { + var err error + ok, err := dbIsPopulated(tx) + if err != nil { + return err + } + // populate the db + if !ok { + for _, stmt := range dbCreateStmts { + _, err = tx.Exec(stmt) + if err != nil { + return err + } + } + return nil + } + // if db is populated check its version + version, err := getDBVersion(tx) + if err != nil { + return err + } + if version < dbVersion { + needsMigrate = true + } + if version > dbVersion { + return fmt.Errorf("Current store db version: %d greater than the current rkt expected version: %d", version, dbVersion) + } + return nil + } + if err = db.Do(fn); err != nil { + return nil, err + } + + // migration is done in another transaction as it must take an exclusive + // store lock. If, in the meantime, another process has already done the + // migration, between the previous db version check and the below + // migration code, the migration will do nothing as it'll start + // migration from the current version. + if needsMigrate { + // Take an exclusive store lock + err := s.storeLock.ExclusiveLock() + if err != nil { + return nil, err + } + // TODO(sgotti) take a db backup (for debugging and last resort rollback?) + fn := func(tx *sql.Tx) error { + return migrate(tx, dbVersion) + } + if err = db.Do(fn); err != nil { + return nil, err + } + } + + return s, nil +} + +// TmpFile returns an *os.File local to the same filesystem as the Store, or +// any error encountered +func (s Store) TmpFile() (*os.File, error) { + dir, err := s.TmpDir() + if err != nil { + return nil, err + } + return ioutil.TempFile(dir, "") +} + +// TmpDir creates and returns dir local to the same filesystem as the Store, +// or any error encountered +func (s Store) TmpDir() (string, error) { + dir := filepath.Join(s.base, "tmp") + if err := os.MkdirAll(dir, defaultPathPerm); err != nil { + return "", err + } + return dir, nil +} + +// ResolveKey resolves a partial key (of format `sha512-0c45e8c0ab2`) to a full +// key by considering the key a prefix and using the store for resolution. +// If the key is longer than the full key length, it is first truncated. +func (s Store) ResolveKey(key string) (string, error) { + if !strings.HasPrefix(key, hashPrefix) { + return "", fmt.Errorf("wrong key prefix") + } + if len(key) < minlenKey { + return "", fmt.Errorf("key too short") + } + if len(key) > lenKey { + key = key[:lenKey] + } + + aciInfos := []*ACIInfo{} + err := s.db.Do(func(tx *sql.Tx) error { + var err error + aciInfos, err = GetACIInfosWithKeyPrefix(tx, key) + return err + }) + if err != nil { + return "", fmt.Errorf("error retrieving ACI Infos: %v", err) + } + + keyCount := len(aciInfos) + if keyCount == 0 { + return "", fmt.Errorf("no keys found") + } + if keyCount != 1 { + return "", fmt.Errorf("ambiguous key: %q", key) + } + return aciInfos[0].BlobKey, nil +} + +func (s Store) ReadStream(key string) (io.ReadCloser, error) { + key, err := s.ResolveKey(key) + if err != nil { + return nil, fmt.Errorf("error resolving key: %v", err) + } + keyLock, err := lock.SharedKeyLock(s.imageLockDir, key) + if err != nil { + return nil, fmt.Errorf("error locking image: %v", err) + } + defer keyLock.Close() + + return s.stores[blobType].ReadStream(key, false) +} + +// WriteACI takes an ACI encapsulated in an io.Reader, decompresses it if +// necessary, and then stores it in the store under a key based on the image ID +// (i.e. the hash of the uncompressed ACI) +// latest defines if the aci has to be marked as the latest. For example an ACI +// discovered without asking for a specific version (latest pattern). +func (s Store) WriteACI(r io.Reader, latest bool) (string, error) { + // Peek at the first 512 bytes of the reader to detect filetype + br := bufio.NewReaderSize(r, 32768) + hd, err := br.Peek(512) + switch err { + case nil: + case io.EOF: // We may have still peeked enough to guess some types, so fall through + default: + return "", fmt.Errorf("error reading image header: %v", err) + } + typ, err := aci.DetectFileType(bytes.NewBuffer(hd)) + if err != nil { + return "", fmt.Errorf("error detecting image type: %v", err) + } + dr, err := decompress(br, typ) + if err != nil { + return "", fmt.Errorf("error decompressing image: %v", err) + } + + // Write the decompressed image (tar) to a temporary file on disk, and + // tee so we can generate the hash + h := sha512.New() + tr := io.TeeReader(dr, h) + fh, err := s.TmpFile() + if err != nil { + return "", fmt.Errorf("error creating image: %v", err) + } + if _, err := io.Copy(fh, tr); err != nil { + return "", fmt.Errorf("error copying image: %v", err) + } + im, err := aci.ManifestFromImage(fh) + if err != nil { + return "", fmt.Errorf("error extracting image manifest: %v", err) + } + if err := fh.Close(); err != nil { + return "", fmt.Errorf("error closing image: %v", err) + } + + // Import the uncompressed image into the store at the real key + key := s.HashToKey(h) + keyLock, err := lock.ExclusiveKeyLock(s.imageLockDir, key) + if err != nil { + return "", fmt.Errorf("error locking image: %v", err) + } + defer keyLock.Close() + + if err = s.stores[blobType].Import(fh.Name(), key, true); err != nil { + return "", fmt.Errorf("error importing image: %v", err) + } + + // Save the imagemanifest using the same key used for the image + imj, err := json.Marshal(im) + if err != nil { + return "", fmt.Errorf("error marshalling image manifest: %v", err) + } + if err = s.stores[imageManifestType].Write(key, imj); err != nil { + return "", fmt.Errorf("error importing image manifest: %v", err) + } + + // Save aciinfo + if err = s.db.Do(func(tx *sql.Tx) error { + aciinfo := &ACIInfo{ + BlobKey: key, + AppName: im.Name.String(), + ImportTime: time.Now(), + Latest: latest, + } + return WriteACIInfo(tx, aciinfo) + }); err != nil { + return "", fmt.Errorf("error writing ACI Info: %v", err) + } + + // The treestore for this ACI is not written here as ACIs downloaded as + // dependencies of another ACI will be exploded also if never directly used. + // Users of treestore should call s.RenderTreeStore before using it. + + return key, nil +} + +// RenderTreeStore renders a treestore for the given image key if it's not +// already fully rendered. +// Users of treestore should call s.RenderTreeStore before using it to ensure +// that the treestore is completely rendered. +func (s Store) RenderTreeStore(key string, rebuild bool) error { + // this lock references the treestore dir for the specified key. This + // is different from a lock on an image key as internally + // treestore.Write calls the acirenderer functions that use GetACI and + // GetImageManifest which are taking the image(s) lock. + treeStoreKeyLock, err := lock.ExclusiveKeyLock(s.treeStoreLockDir, key) + if err != nil { + return fmt.Errorf("error locking tree store: %v", err) + } + defer treeStoreKeyLock.Close() + + if !rebuild { + rendered, err := s.treestore.IsRendered(key) + if err != nil { + return fmt.Errorf("cannot determine if tree is already rendered: %v", err) + } + if rendered { + return nil + } + } + // Firstly remove a possible partial treestore if existing. + // This is needed as a previous ACI removal operation could have failed + // cleaning the tree store leaving some stale files. + err = s.treestore.Remove(key) + if err != nil { + return err + } + err = s.treestore.Write(key, &s) + if err != nil { + return err + } + return nil +} + +// CheckTreeStore verifies the treestore consistency for the specified key. +func (s Store) CheckTreeStore(key string) error { + treeStoreKeyLock, err := lock.SharedKeyLock(s.treeStoreLockDir, key) + if err != nil { + return fmt.Errorf("error locking tree store: %v", err) + } + defer treeStoreKeyLock.Close() + + return s.treestore.Check(key) +} + +// GetTreeStorePath returns the absolute path of the treestore for the specified key. +// It doesn't ensure that the path exists and is fully rendered. This should +// be done calling IsRendered() +func (s Store) GetTreeStorePath(key string) string { + return s.treestore.GetPath(key) +} + +// GetTreeStoreRootFS returns the absolute path of the rootfs in the treestore +// for specified key. +// It doesn't ensure that the rootfs exists and is fully rendered. This should +// be done calling IsRendered() +func (s Store) GetTreeStoreRootFS(key string) string { + return s.treestore.GetRootFS(key) +} + +// GetRemote tries to retrieve a remote with the given ACIURL. found will be +// false if remote doesn't exist. +func (s Store) GetRemote(aciURL string) (*Remote, bool, error) { + var remote *Remote + found := false + err := s.db.Do(func(tx *sql.Tx) error { + var err error + remote, found, err = GetRemote(tx, aciURL) + return err + }) + return remote, found, err +} + +// WriteRemote adds or updates the provided Remote. +func (s Store) WriteRemote(remote *Remote) error { + err := s.db.Do(func(tx *sql.Tx) error { + return WriteRemote(tx, remote) + }) + return err +} + +// Get the ImageManifest with the specified key. +func (s Store) GetImageManifest(key string) (*schema.ImageManifest, error) { + key, err := s.ResolveKey(key) + if err != nil { + return nil, fmt.Errorf("error resolving key: %v", err) + } + keyLock, err := lock.SharedKeyLock(s.imageLockDir, key) + if err != nil { + return nil, fmt.Errorf("error locking image: %v", err) + } + defer keyLock.Close() + + imj, err := s.stores[imageManifestType].Read(key) + if err != nil { + return nil, fmt.Errorf("error retrieving image manifest: %v", err) + } + var im *schema.ImageManifest + if err = json.Unmarshal(imj, &im); err != nil { + return nil, fmt.Errorf("error unmarshalling image manifest: %v", err) + } + return im, nil +} + +// GetACI retrieves the ACI that best matches the provided app name and labels. +// The returned value is the blob store key of the retrieved ACI. +// If there are multiple matching ACIs choose the latest one (defined as the +// last one imported in the store). +// If no version label is requested, ACIs marked as latest in the ACIInfo are +// preferred. +func (s Store) GetACI(name types.ACName, labels types.Labels) (string, error) { + var curaciinfo *ACIInfo + versionRequested := false + if _, ok := labels.Get("version"); ok { + versionRequested = true + } + + var aciinfos []*ACIInfo + err := s.db.Do(func(tx *sql.Tx) error { + var err error + aciinfos, _, err = GetACIInfosWithAppName(tx, name.String()) + return err + }) + if err != nil { + return "", err + } + +nextKey: + for _, aciinfo := range aciinfos { + im, err := s.GetImageManifest(aciinfo.BlobKey) + if err != nil { + return "", fmt.Errorf("error getting image manifest: %v", err) + } + + // The image manifest must have all the requested labels + for _, l := range labels { + ok := false + for _, rl := range im.Labels { + if l.Name == rl.Name && l.Value == rl.Value { + ok = true + break + } + } + if !ok { + continue nextKey + } + } + + if curaciinfo != nil { + // If no version is requested prefer the acis marked as latest + if !versionRequested { + if !curaciinfo.Latest && aciinfo.Latest { + curaciinfo = aciinfo + continue nextKey + } + if curaciinfo.Latest && !aciinfo.Latest { + continue nextKey + } + } + // If multiple matching image manifests are found, choose the latest imported in the cas. + if aciinfo.ImportTime.After(curaciinfo.ImportTime) { + curaciinfo = aciinfo + } + } else { + curaciinfo = aciinfo + } + } + + if curaciinfo != nil { + return curaciinfo.BlobKey, nil + } + return "", fmt.Errorf("aci not found") +} + +func (s Store) Dump(hex bool) { + for _, s := range s.stores { + var keyCount int + for key := range s.Keys(nil) { + val, err := s.Read(key) + if err != nil { + panic(fmt.Sprintf("key %s had no value", key)) + } + if len(val) > 128 { + val = val[:128] + } + out := string(val) + if hex { + out = fmt.Sprintf("%x", val) + } + fmt.Printf("%s/%s: %s\n", s.BasePath, key, out) + keyCount++ + } + fmt.Printf("%d total keys\n", keyCount) + } +} + +// HashToKey takes a hash.Hash (which currently _MUST_ represent a full SHA512), +// calculates its sum, and returns a string which should be used as the key to +// store the data matching the hash. +func (s Store) HashToKey(h hash.Hash) string { + return hashToKey(h) +} + +func hashToKey(h hash.Hash) string { + s := h.Sum(nil) + return keyToString(s) +} + +// keyToString takes a key and returns a shortened and prefixed hexadecimal string version +func keyToString(k []byte) string { + if len(k) != lenHash { + panic(fmt.Sprintf("bad hash passed to hashToKey: %x", k)) + } + return fmt.Sprintf("%s%x", hashPrefix, k)[0:lenKey] +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/store_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/store_test.go new file mode 100644 index 00000000000..2ee2b9707ed --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/store_test.go @@ -0,0 +1,453 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "archive/tar" + "bytes" + "database/sql" + "encoding/hex" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/appc/spec/schema/types" + "github.com/coreos/rkt/pkg/aci" +) + +const tstprefix = "store-test" + +func TestBlobStore(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + for _, valueStr := range []string{ + "I am a manually placed object", + } { + s.stores[blobType].Write(types.NewHashSHA512([]byte(valueStr)).String(), []byte(valueStr)) + } + + s.Dump(false) +} + +func TestResolveKey(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Return a hash key buffer from a hex string + str2key := func(s string) *bytes.Buffer { + k, _ := hex.DecodeString(s) + return bytes.NewBufferString(keyToString(k)) + } + + // Set up store (use key == data for simplicity) + data := []*bytes.Buffer{ + str2key("12345678900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + str2key("abcdefabc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + str2key("abcabcabc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + str2key("abc01234500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + str2key("67147019a5b56f5e2ee01e989a8aa4787f56b8445960be2d8678391cf111009bc0780f31001fd181a2b61507547aee4caa44cda4b8bdb238d0e4ba830069ed2c"), + } + for _, d := range data { + // Save aciinfo + err := s.db.Do(func(tx *sql.Tx) error { + aciinfo := &ACIInfo{ + BlobKey: d.String(), + AppName: "example.com/app", + ImportTime: time.Now(), + } + return WriteACIInfo(tx, aciinfo) + }) + if err != nil { + t.Fatalf("error writing to store: %v", err) + } + } + + // Full key already - should return short version of the full key + fkl := "sha512-67147019a5b56f5e2ee01e989a8aa4787f56b8445960be2d8678391cf111009bc0780f31001fd181a2b61507547aee4caa44cda4b8bdb238d0e4ba830069ed2c" + fks := "sha512-67147019a5b56f5e2ee01e989a8aa4787f56b8445960be2d8678391cf111009b" + for _, k := range []string{fkl, fks} { + key, err := s.ResolveKey(k) + if key != fks { + t.Errorf("expected ResolveKey to return unaltered short key, but got %q", key) + } + if err != nil { + t.Errorf("expected err=nil, got %v", err) + } + } + + // Unambiguous prefix match + k, err := s.ResolveKey("sha512-123") + if k != "sha512-1234567890000000000000000000000000000000000000000000000000000000" { + t.Errorf("expected %q, got %q", "sha512-1234567890000000000000000000000000000000000000000000000000000000", k) + } + if err != nil { + t.Errorf("expected err=nil, got %v", err) + } + + // Ambiguous prefix match + k, err = s.ResolveKey("sha512-abc") + if k != "" { + t.Errorf("expected %q, got %q", "", k) + } + if err == nil { + t.Errorf("expected non-nil error!") + } + + // wrong key prefix + k, err = s.ResolveKey("badprefix-1") + expectedErr := "wrong key prefix" + if err == nil { + t.Errorf("expected non-nil error!") + } + if err.Error() != expectedErr { + t.Errorf("expected err=%q, got %q", expectedErr, err) + } + + // key too short + k, err = s.ResolveKey("sha512-1") + expectedErr = "key too short" + if err == nil { + t.Errorf("expected non-nil error!") + } + if err.Error() != expectedErr { + t.Errorf("expected err=%q, got %q", expectedErr, err) + } +} + +func TestGetImageManifest(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + imj := `{ + "acKind": "ImageManifest", + "acVersion": "0.5.4", + "name": "example.com/test01" + }` + + aci, err := aci.NewACI(dir, imj, nil) + if err != nil { + t.Fatalf("error creating test tar: %v", err) + } + // Rewind the ACI + if _, err := aci.Seek(0, 0); err != nil { + t.Fatalf("unexpected error %v", err) + } + key, err := s.WriteACI(aci, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wanted := "example.com/test01" + im, err := s.GetImageManifest(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if im.Name.String() != wanted { + t.Errorf("expected im with name: %s, got: %s", wanted, im.Name.String()) + } + + // test unexistent key + im, err = s.GetImageManifest("sha512-aaaaaaaaaaaaaaaaa") + if err == nil { + t.Fatalf("expected non-nil error!") + } +} + +func TestGetAci(t *testing.T) { + type test struct { + name types.ACName + labels types.Labels + expected int // the aci index to expect or -1 if not result expected, + } + + type acidef struct { + imj string + latest bool + } + + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + tests := []struct { + acidefs []acidef + tests []test + }{ + { + []acidef{ + { + `{ + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test01" + }`, + false, + }, + { + `{ + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02", + "labels": [ + { + "name": "version", + "value": "1.0.0" + } + ] + }`, + true, + }, + { + `{ + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": "example.com/test02", + "labels": [ + { + "name": "version", + "value": "2.0.0" + } + ] + }`, + false, + }, + }, + []test{ + { + "example.com/unexistentaci", + types.Labels{}, + -1, + }, + { + "example.com/test01", + types.Labels{}, + 0, + }, + { + "example.com/test02", + types.Labels{ + { + Name: "version", + Value: "1.0.0", + }, + }, + 1, + }, + { + "example.com/test02", + types.Labels{ + { + Name: "version", + Value: "2.0.0", + }, + }, + 2, + }, + { + "example.com/test02", + types.Labels{}, + 1, + }, + }, + }, + } + + for _, tt := range tests { + keys := []string{} + // Create ACIs + for _, ad := range tt.acidefs { + aci, err := aci.NewACI(dir, ad.imj, nil) + if err != nil { + t.Fatalf("error creating test tar: %v", err) + } + + // Rewind the ACI + if _, err := aci.Seek(0, 0); err != nil { + t.Fatalf("unexpected error %v", err) + } + + key, err := s.WriteACI(aci, ad.latest) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + keys = append(keys, key) + } + + for _, test := range tt.tests { + key, err := s.GetACI(test.name, test.labels) + if test.expected == -1 { + if err == nil { + t.Fatalf("Expected no key for appName %s, got %s", test.name, key) + } + + } else { + if err != nil { + t.Fatalf("unexpected error on GetACI for name %s, labels: %v: %v", test.name, test.labels, err) + } + if keys[test.expected] != key { + t.Errorf("expected key: %s, got %s. GetACI with name: %s, labels: %v", key, keys[test.expected], test.name, test.labels) + } + } + } + } +} + +func TestTreeStore(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.5.4", + "name": "example.com/test01" + } + ` + + entries := []*aci.ACIEntry{ + // An empty dir + { + Header: &tar.Header{ + Name: "rootfs/a", + Typeflag: tar.TypeDir, + }, + }, + { + Contents: "hello", + Header: &tar.Header{ + Name: "hello.txt", + Size: 5, + }, + }, + { + Header: &tar.Header{ + Name: "rootfs/link.txt", + Linkname: "rootfs/hello.txt", + Typeflag: tar.TypeSymlink, + }, + }, + // dangling symlink + { + Header: &tar.Header{ + Name: "rootfs/link2.txt", + Linkname: "rootfs/missingfile.txt", + Typeflag: tar.TypeSymlink, + }, + }, + { + Header: &tar.Header{ + Name: "rootfs/fifo", + Typeflag: tar.TypeFifo, + }, + }, + } + aci, err := aci.NewACI(dir, imj, entries) + if err != nil { + t.Fatalf("error creating test tar: %v", err) + } + defer aci.Close() + + // Rewind the ACI + if _, err := aci.Seek(0, 0); err != nil { + t.Fatalf("unexpected error %v", err) + } + + // Import the new ACI + key, err := s.WriteACI(aci, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Ask the store to render the treestore + err = s.RenderTreeStore(key, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify image Hash. Should be the same. + err = s.CheckTreeStore(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Change a file permission + rootfs := s.GetTreeStoreRootFS(key) + err = os.Chmod(filepath.Join(rootfs, "a"), 0600) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify image Hash. Should be different + err = s.CheckTreeStore(key) + if err == nil { + t.Fatalf("unexpected error: %v", err) + } + + // rebuild the tree + err = s.RenderTreeStore(key, true) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Add a file + rootfs = s.GetTreeStoreRootFS(key) + err = ioutil.WriteFile(filepath.Join(rootfs, "newfile"), []byte("newfile"), 0644) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify image Hash. Should be different + err = s.CheckTreeStore(key) + if err == nil { + t.Fatalf("unexpected error: %v", err) + } + +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/tree.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/tree.go new file mode 100644 index 00000000000..96f780912a5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/tree.go @@ -0,0 +1,337 @@ +package store + +import ( + "archive/tar" + "crypto/sha512" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "syscall" + + specaci "github.com/appc/spec/aci" + "github.com/appc/spec/pkg/tarheader" + "github.com/appc/spec/schema/types" + "github.com/coreos/rkt/pkg/aci" + "github.com/coreos/rkt/pkg/sys" +) + +const ( + hashfilename = "hash" + renderedfilename = "rendered" +) + +// TreeStore represents a store of rendered ACIs +// The image's key becomes the name of the directory containing the rendered aci. +type TreeStore struct { + path string +} + +// Write renders the ACI with the provided key in the treestore +// Write, to avoid having a rendered ACI with old stale files, requires that +// the destination directory doesn't exist (usually Remove should be called +// before Write) +func (ts *TreeStore) Write(key string, s *Store) error { + treepath := filepath.Join(ts.path, key) + fi, _ := os.Stat(treepath) + if fi != nil { + return fmt.Errorf("treestore: path %s already exists", treepath) + } + imageID, err := types.NewHash(key) + if err != nil { + return fmt.Errorf("treestore: cannot convert key to imageID: %v", err) + } + err = aci.RenderACIWithImageID(*imageID, treepath, s) + if err != nil { + return fmt.Errorf("treestore: cannot render aci: %v", err) + } + hash, err := ts.Hash(key) + if err != nil { + return fmt.Errorf("treestore: cannot calculate tree hash: %v", err) + } + err = ioutil.WriteFile(filepath.Join(treepath, hashfilename), []byte(hash), 0644) + if err != nil { + return fmt.Errorf("treestore: cannot write hash file: %v", err) + } + // before creating the "rendered" flag file we need to ensure that all data is fsynced + dfd, err := syscall.Open(treepath, syscall.O_RDONLY, 0) + if err != nil { + return err + } + defer syscall.Close(dfd) + if err := sys.Syncfs(dfd); err != nil { + return fmt.Errorf("treestore: failed to sync data: %v", err) + } + // Create rendered file + f, err := os.Create(filepath.Join(treepath, renderedfilename)) + if err != nil { + return fmt.Errorf("treestore: failed to write rendered file: %v", err) + } + f.Close() + + if err := syscall.Fsync(dfd); err != nil { + return fmt.Errorf("treestore: failed to sync tree store directory: %v", err) + } + return nil +} + +// Remove cleans the directory for the specified key +func (ts *TreeStore) Remove(key string) error { + treepath := filepath.Join(ts.path, key) + // If tree path doesn't exist we're done + _, err := os.Stat(treepath) + if err != nil && os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("treestore: failed to open tree store directory: %v", err) + } + + renderedFilePath := filepath.Join(treepath, renderedfilename) + // The "rendered" flag file should be the firstly removed file. So if + // the removal ends with some error leaving some stale files IsRendered() + // will return false. + _, err = os.Stat(renderedFilePath) + if err != nil && !os.IsNotExist(err) { + return err + } + if !os.IsNotExist(err) { + err := os.Remove(renderedFilePath) + // Ensure that the treepath directory is fsynced after removing the + // "rendered" flag file + f, err := os.Open(treepath) + if err != nil { + return fmt.Errorf("treestore: failed to open tree store directory: %v", err) + } + defer f.Close() + err = f.Sync() + if err != nil { + return fmt.Errorf("treestore: failed to sync tree store directory: %v", err) + } + } + return os.RemoveAll(treepath) +} + +// IsRendered checks if the tree store is fully rendered +func (ts *TreeStore) IsRendered(key string) (bool, error) { + // if the "rendered" flag file exists, assume that the store is already + // fully rendered. + treepath := filepath.Join(ts.path, key) + _, err := os.Stat(filepath.Join(treepath, renderedfilename)) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// GetPath returns the absolute path of the treestore for the specified key. +// It doesn't ensure that the path exists and is fully rendered. This should +// be done calling IsRendered() +func (ts *TreeStore) GetPath(key string) string { + return filepath.Join(ts.path, key) +} + +// GetRootFS returns the absolute path of the rootfs for the specified key. +// It doesn't ensure that the rootfs exists and is fully rendered. This should +// be done calling IsRendered() +func (ts *TreeStore) GetRootFS(key string) string { + return filepath.Join(ts.GetPath(key), "rootfs") +} + +// TreeStore calculates an hash of the rendered ACI. It uses the same functions +// used to create a tar but instead of writing the full archive is just +// computes the sha512 sum of the file infos and contents. +func (ts *TreeStore) Hash(key string) (string, error) { + treepath := filepath.Join(ts.path, key) + + hash := sha512.New() + iw := NewHashWriter(hash) + err := filepath.Walk(treepath, buildWalker(treepath, iw)) + if err != nil { + return "", fmt.Errorf("treestore: error walking rootfs: %v", err) + } + + hashstring := hashToKey(hash) + + return hashstring, nil +} + +// Check calculates the actual rendered ACI's hash and verifies that it matches +// the saved value. +func (ts *TreeStore) Check(key string) error { + treepath := filepath.Join(ts.path, key) + hash, err := ioutil.ReadFile(filepath.Join(treepath, hashfilename)) + if err != nil { + return fmt.Errorf("treestore: cannot read hash file: %v", err) + } + curhash, err := ts.Hash(key) + if err != nil { + return fmt.Errorf("treestore: cannot calculate tree hash: %v", err) + } + if curhash != string(hash) { + return fmt.Errorf("treestore: wrong tree hash: %s, expected: %s", curhash, hash) + } + return nil +} + +type xattr struct { + Name string + Value string +} + +// Like tar Header but, to keep json output reproducible: +// * Xattrs as a slice +// * Skip Uname and Gname +// TODO. Should ModTime/AccessTime/ChangeTime be saved? For validation its +// probably enough to hash the file contents and the other infos and avoid +// problems due to them changing. +// TODO(sgotti) Is it possible that json output will change between go +// versions? Use another or our own Marshaller? +type fileInfo struct { + Name string // name of header file entry + Mode int64 // permission and mode bits + Uid int // user id of owner + Gid int // group id of owner + Size int64 // length in bytes + Typeflag byte // type of header entry + Linkname string // target name of link + Devmajor int64 // major number of character or block device + Devminor int64 // minor number of character or block device + Xattrs []xattr +} + +func FileInfoFromHeader(hdr *tar.Header) *fileInfo { + fi := &fileInfo{ + Name: hdr.Name, + Mode: hdr.Mode, + Uid: hdr.Uid, + Gid: hdr.Gid, + Size: hdr.Size, + Typeflag: hdr.Typeflag, + Linkname: hdr.Linkname, + Devmajor: hdr.Devmajor, + Devminor: hdr.Devminor, + } + keys := make([]string, len(hdr.Xattrs)) + for k := range hdr.Xattrs { + keys = append(keys, k) + } + sort.Strings(keys) + + xattrs := make([]xattr, 0) + for _, k := range keys { + xattrs = append(xattrs, xattr{Name: k, Value: hdr.Xattrs[k]}) + } + fi.Xattrs = xattrs + return fi +} + +// TODO(sgotti) this func is copied from appcs/spec/aci/build.go but also +// removes the hashfile and the renderedfile. Find a way to reuse it. +func buildWalker(root string, aw specaci.ArchiveWriter) filepath.WalkFunc { + // cache of inode -> filepath, used to leverage hard links in the archive + inos := map[uint64]string{} + return func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relpath, err := filepath.Rel(root, path) + if err != nil { + return err + } + if relpath == "." { + return nil + } + if relpath == specaci.ManifestFile || relpath == hashfilename || relpath == renderedfilename { + // ignore; this will be written by the archive writer + // TODO(jonboulle): does this make sense? maybe just remove from archivewriter? + return nil + } + + link := "" + var r io.Reader + switch info.Mode() & os.ModeType { + case os.ModeSocket: + return nil + case os.ModeNamedPipe: + case os.ModeCharDevice: + case os.ModeDevice: + case os.ModeDir: + case os.ModeSymlink: + target, err := os.Readlink(path) + if err != nil { + return err + } + link = target + default: + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + r = file + } + + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + panic(err) + } + // Because os.FileInfo's Name method returns only the base + // name of the file it describes, it may be necessary to + // modify the Name field of the returned header to provide the + // full path name of the file. + hdr.Name = relpath + tarheader.Populate(hdr, info, inos) + // If the file is a hard link to a file we've already seen, we + // don't need the contents + if hdr.Typeflag == tar.TypeLink { + hdr.Size = 0 + r = nil + } + + if err := aw.AddFile(hdr, r); err != nil { + return err + } + return nil + } +} + +type imageHashWriter struct { + io.Writer +} + +func NewHashWriter(w io.Writer) specaci.ArchiveWriter { + return &imageHashWriter{w} +} + +func (aw *imageHashWriter) AddFile(hdr *tar.Header, r io.Reader) error { + // Write the json encoding of the FileInfo struct + hdrj, err := json.Marshal(FileInfoFromHeader(hdr)) + if err != nil { + return err + } + _, err = aw.Writer.Write(hdrj) + if err != nil { + return err + } + + if r != nil { + // Write the file data + _, err := io.Copy(aw.Writer, r) + if err != nil { + return err + } + } + + return nil +} + +func (aw *imageHashWriter) Close() error { + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/tree_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/tree_test.go new file mode 100644 index 00000000000..4e434fd7a16 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/tree_test.go @@ -0,0 +1,138 @@ +package store + +import ( + "archive/tar" + "io/ioutil" + "os" + "testing" + + "github.com/coreos/rkt/pkg/aci" +) + +func treeStoreWriteACI(dir string, s *Store) (string, error) { + imj := ` + { + "acKind": "ImageManifest", + "acVersion": "0.5.4", + "name": "example.com/test01" + } + ` + + entries := []*aci.ACIEntry{ + // An empty dir + { + Header: &tar.Header{ + Name: "rootfs/a", + Typeflag: tar.TypeDir, + }, + }, + { + Contents: "hello", + Header: &tar.Header{ + Name: "hello.txt", + Size: 5, + }, + }, + { + Header: &tar.Header{ + Name: "rootfs/link.txt", + Linkname: "rootfs/hello.txt", + Typeflag: tar.TypeSymlink, + }, + }, + // dangling symlink + { + Header: &tar.Header{ + Name: "rootfs/link2.txt", + Linkname: "rootfs/missingfile.txt", + Typeflag: tar.TypeSymlink, + }, + }, + { + Header: &tar.Header{ + Name: "rootfs/fifo", + Typeflag: tar.TypeFifo, + }, + }, + } + aci, err := aci.NewACI(dir, imj, entries) + if err != nil { + return "", err + } + defer aci.Close() + + // Rewind the ACI + if _, err := aci.Seek(0, 0); err != nil { + return "", err + } + + // Import the new ACI + key, err := s.WriteACI(aci, false) + if err != nil { + return "", err + } + return key, nil +} + +func TestTreeStoreWrite(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + key, err := treeStoreWriteACI(dir, s) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Ask the store to render the treestore + err = s.treestore.Write(key, s) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify image Hash. Should be the same. + err = s.treestore.Check(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestTreeStoreRemove(t *testing.T) { + dir, err := ioutil.TempDir("", tstprefix) + if err != nil { + t.Fatalf("error creating tempdir: %v", err) + } + defer os.RemoveAll(dir) + s, err := NewStore(dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + key, err := treeStoreWriteACI(dir, s) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Test non existent dir + err = s.treestore.Remove(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Test rendered tree + err = s.treestore.Write(key, s) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + err = s.treestore.Remove(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/utils.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/utils.go new file mode 100644 index 00000000000..74191c27dbf --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/utils.go @@ -0,0 +1,74 @@ +// Copyright 2014 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "compress/bzip2" + "compress/gzip" + "errors" + "fmt" + "io" + "net/url" + "strings" + + "github.com/appc/spec/aci" +) + +// blockTransform creates a path slice from the given string to use as a +// directory prefix. The string must be in hash format: +// "sha256-abcdefgh"... -> []{"sha256", "ab"} +// Right now it just copies the default of git which is a two byte prefix. We +// will likely want to add re-sharding later. +func blockTransform(s string) []string { + // TODO(philips): use spec/types.Hash after export typ field + parts := strings.SplitN(s, "-", 2) + if len(parts) != 2 { + panic(fmt.Errorf("blockTransform should never receive non-hash, got %v", s)) + } + return []string{ + parts[0], + parts[1][0:2], + } +} + +func parseAlways(s string) *url.URL { + u, _ := url.Parse(s) + return u +} + +func decompress(rs io.Reader, typ aci.FileType) (io.Reader, error) { + var ( + dr io.Reader + err error + ) + switch typ { + case aci.TypeGzip: + dr, err = gzip.NewReader(rs) + if err != nil { + return nil, err + } + case aci.TypeBzip2: + dr = bzip2.NewReader(rs) + case aci.TypeXz: + dr = aci.XzReader(rs) + case aci.TypeTar: + dr = rs + case aci.TypeUnknown: + return nil, errors.New("error: unknown image filetype") + default: + return nil, errors.New("no type returned from DetectFileType?") + } + return dr, nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/rkt/store/z_last_test.go b/Godeps/_workspace/src/github.com/coreos/rkt/store/z_last_test.go new file mode 100644 index 00000000000..0724e253b52 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/rkt/store/z_last_test.go @@ -0,0 +1,59 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package store + +import ( + "runtime" + "sort" + "strings" + "testing" +) + +func interestingGoroutines() (gs []string) { + buf := make([]byte, 2<<20) + buf = buf[:runtime.Stack(buf, true)] + for _, g := range strings.Split(string(buf), "\n\n") { + sl := strings.SplitN(g, "\n", 2) + if len(sl) != 2 { + continue + } + stack := strings.TrimSpace(sl[1]) + if stack == "" || + strings.Contains(stack, "testing.RunTests") || + strings.Contains(stack, "testing.Main(") || + strings.Contains(stack, "runtime.goexit") || + strings.Contains(stack, "created by runtime.gc") || + strings.Contains(stack, "runtime.MHeap_Scavenger") { + continue + } + gs = append(gs, stack) + } + sort.Strings(gs) + return +} + +// Verify the other tests didn't leave any goroutines running. +// This is in a file named z_last_test.go so it sorts at the end. +func TestGoroutinesRunning(t *testing.T) { + if testing.Short() { + t.Skip("not counting goroutines for leakage in -short mode") + } + gs := interestingGoroutines() + + n := 0 + stackCount := make(map[string]int) + for _, g := range gs { + stackCount[g]++ + n++ + } + + t.Logf("num goroutines = %d", n) + if n > 0 { + t.Error("Too many goroutines.") + for stack, count := range stackCount { + t.Logf("%d instances of:\n%s", count, stack) + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/bufs/AUTHORS b/Godeps/_workspace/src/github.com/cznic/bufs/AUTHORS new file mode 100644 index 00000000000..0078f5f5b6d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/bufs/AUTHORS @@ -0,0 +1,11 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/bufs/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/bufs/CONTRIBUTORS new file mode 100644 index 00000000000..5e86f0607ce --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/bufs/CONTRIBUTORS @@ -0,0 +1,9 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/bufs/LICENSE b/Godeps/_workspace/src/github.com/cznic/bufs/LICENSE new file mode 100644 index 00000000000..7d80fe28eb1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/bufs/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The bufs Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/bufs/Makefile b/Godeps/_workspace/src/github.com/cznic/bufs/Makefile new file mode 100644 index 00000000000..c98c23d6dfb --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/bufs/Makefile @@ -0,0 +1,31 @@ +# Copyright 2014 The bufs Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +all: clean + go fmt + go test -i + go test + go build + go vet + golint . + go install + make todo + +todo: + @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alnum:]] *.go || true + @grep -n TODO *.go || true + @grep -n FIXME *.go || true + @grep -n BUG *.go || true + +clean: + rm -f bufs.test mem.out *~ + +demo: + go test -bench . -benchmem + go test -c + ./bufs.test -test.v -test.run Foo -test.memprofile mem.out \ + -test.memprofilerate 1 + go tool pprof bufs.test mem.out --alloc_space --nodefraction 0.0001 \ + --edgefraction 0 -web + @echo "Note: Foo vs FooBufs allocated memory is in hundreds of MBs vs 8 kB." diff --git a/Godeps/_workspace/src/github.com/cznic/bufs/README.md b/Godeps/_workspace/src/github.com/cznic/bufs/README.md new file mode 100644 index 00000000000..3f3d56f19d5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/bufs/README.md @@ -0,0 +1,8 @@ +bufs +==== + +Package bufs implements a simple buffer cache. + + installation: go get github.com/cznic/bufs + +documentation: http://godoc.org/github.com/cznic/bufs diff --git a/Godeps/_workspace/src/github.com/cznic/bufs/bufs.go b/Godeps/_workspace/src/github.com/cznic/bufs/bufs.go new file mode 100644 index 00000000000..f4e0eee2a6f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/bufs/bufs.go @@ -0,0 +1,391 @@ +// Copyright 2014 The bufs Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bufs implements a simple buffer cache. +// +// The intended use scheme is like: +// +// type Foo struct { +// buffers bufs.Buffers +// ... +// } +// +// // Bar can call Qux, but not the other way around (in this example). +// const maxFooDepth = 2 +// +// func NewFoo() *Foo { +// return &Foo{buffers: bufs.New(maxFooDepth), ...} +// } +// +// func (f *Foo) Bar(n int) { +// buf := f.buffers.Alloc(n) // needed locally for computation and/or I/O +// defer f.buffers.Free() +// ... +// f.Qux(whatever) +// } +// +// func (f *Foo) Qux(n int) { +// buf := f.buffers.Alloc(n) // needed locally for computation and/or I/O +// defer f.buffers.Free() +// ... +// } +// +// The whole idea behind 'bufs' is that when calling e.g. Foo.Bar N times, then +// normally, without using 'bufs', there will be 2*N (in this example) []byte +// buffers allocated. While using 'bufs', only 2 buffers (in this example) +// will ever be created. For large N it can be a substantial difference. +// +// It's not a good idea to use Buffers to cache too big buffers. The cost of +// having a cached buffer is that the buffer is naturally not eligible for +// garbage collection. Of course, that holds only while the Foo instance is +// reachable, in the above example. +// +// The buffer count limit is intentionally "hard" (read panicking), although +// configurable in New(). The rationale is to prevent recursive calls, using +// Alloc, to cause excessive, "static" memory consumption. Tune the limit +// carefully or do not use Buffers from within [mutually] recursive functions +// where the nesting depth is not realistically bounded to some rather small +// number. +// +// Buffers cannot guarantee improvements to you program performance. There may +// be a gain in case where they fit well. Firm grasp on what your code is +// actually doing, when and in what order is essential to proper use of +// Buffers. It's _highly_ recommended to first do profiling and memory +// profiling before even thinking about using 'bufs'. The real world example, +// and cause for this package, was a first correct, yet no optimizations done +// version of a program; producing few MB of useful data while allocating 20+GB +// of memory. Of course the garbage collector properly kicked in, yet the +// memory abuse caused ~80+% of run time to be spent memory management. The +// program _was_ expected to be slow in its still development phase, but the +// bottleneck was guessed to be in I/O. Actually the hard disk was waiting for +// the billions bytes being allocated and zeroed. Garbage collect on low +// memory, rinse and repeat. +// +// In the provided tests, TestFoo and TestFooBufs do the same simulated work, +// except the later uses Buffers while the former does not. Suggested test runs +// which show the differences: +// +// $ go test -bench . -benchmem +// +// or +// +// $ go test -c +// $ ./bufs.test -test.v -test.run Foo -test.memprofile mem.out -test.memprofilerate 1 +// $ go tool pprof bufs.test mem.out --alloc_space --nodefraction 0.0001 --edgefraction 0 -web +// $ # Note: Foo vs FooBufs allocated memory is in hundreds of MBs vs 8 kB. +// +// or +// +// $ make demo # same as all of the above +// +// +// NOTE: Alloc/Free calls must be properly nested in the same way as in for +// example BeginTransaction/EndTransaction pairs. If your code can panic then +// the pairing should be enforced by deferred calls. +// +// NOTE: Buffers objects do not allocate any space until requested by Alloc, +// the mechanism works on demand only. +// +// FAQ: Why the 'bufs' package name? +// +// Package name 'bufs' was intentionally chosen instead of the perhaps more +// conventional 'buf'. There are already too many 'buf' named things in the +// code out there and that'll be a source of a lot of trouble. It's a bit +// similar situation as in the case of package "strings" (not "string"). +package bufs + +import ( + "errors" + "sort" + "sync" +) + +// Buffers type represents a buffer ([]byte) cache. +// +// NOTE: Do not modify Buffers directly, use only its methods. Do not create +// additional values (copies) of Buffers, that'll break its functionality. Use +// a pointer instead to refer to a single instance from different +// places/scopes. +type Buffers [][]byte + +// New returns a newly created instance of Buffers with a maximum capacity of n +// buffers. +// +// NOTE: 'bufs.New(n)' is the same as 'make(bufs.Buffers, n)'. +func New(n int) Buffers { + return make(Buffers, n) +} + +// Alloc will return a buffer such that len(r) == n. It will firstly try to +// find an existing and unused buffer of big enough size. Only when there is no +// such, then one of the buffer slots is reallocated to a bigger size. +// +// It's okay to use append with buffers returned by Alloc. But it can cause +// allocation in that case and will again be producing load for the garbage +// collector. The best use of Alloc is for I/O buffers where the needed size of +// the buffer is figured out at some point of the code path in a 'final size' +// sense. Another real world example are compression/decompression buffers. +// +// NOTE: The buffer returned by Alloc _is not_ zeroed. That's okay for e.g. +// passing a buffer to io.Reader. If you need a zeroed buffer use Calloc. +// +// NOTE: Buffers returned from Alloc _must not_ be exposed/returned to your +// clients. Those buffers are intended to be used strictly internally, within +// the methods of some "object". +// +// NOTE: Alloc will panic if there are no buffers (buffer slots) left. +func (p *Buffers) Alloc(n int) (r []byte) { + b := *p + if len(b) == 0 { + panic(errors.New("Buffers.Alloc: out of buffers")) + } + + biggest, best, biggestI, bestI := -1, -1, -1, -1 + for i, v := range b { + //ln := len(v) + // The above was correct, buts it's just confusing. It worked + // because not the buffers, but slices of them are returned in + // the 'if best >= n' code path. + ln := cap(v) + + if ln >= biggest { + biggest, biggestI = ln, i + } + + if ln >= n && (bestI < 0 || best > ln) { + best, bestI = ln, i + if ln == n { + break + } + } + } + + last := len(b) - 1 + if best >= n { + r = b[bestI] + b[last], b[bestI] = b[bestI], b[last] + *p = b[:last] + return r[:n] + } + + r = make([]byte, n, overCommit(n)) + b[biggestI] = r + b[last], b[biggestI] = b[biggestI], b[last] + *p = b[:last] + return +} + +// Calloc will acquire a buffer using Alloc and then clears it to zeros. The +// zeroing goes up to n, not cap(r). +func (p *Buffers) Calloc(n int) (r []byte) { + r = p.Alloc(n) + for i := range r { + r[i] = 0 + } + return +} + +// Free makes the lastly allocated by Alloc buffer free (available) again for +// Alloc. +// +// NOTE: Improper Free invocations, like in the sequence {New, Alloc, Free, +// Free}, will panic. +func (p *Buffers) Free() { + b := *p + b = b[:len(b)+1] + *p = b +} + +// Stats reports memory consumed by Buffers, without accounting for some +// (smallish) additional overhead. +func (p *Buffers) Stats() (bytes int) { + b := *p + b = b[:cap(b)] + for _, v := range b { + bytes += cap(v) + } + return +} + +// Cache caches buffers ([]byte). A zero value of Cache is ready for use. +// +// NOTE: Do not modify a Cache directly, use only its methods. Do not create +// additional values (copies) of a Cache, that'll break its functionality. Use +// a pointer instead to refer to a single instance from different +// places/scopes. +type Cache [][]byte + +// Get returns a buffer ([]byte) of length n. If no such buffer is cached then +// a biggest cached buffer is resized to have length n and returned. If there +// are no cached items at all, Get returns a newly allocated buffer. +// +// In other words the cache policy is: +// +// - If the cache is empty, the buffer must be newly created and returned. +// Cache remains empty. +// +// - If a buffer of sufficient size is found in the cache, remove it from the +// cache and return it. +// +// - Otherwise the cache is non empty, but no cached buffer is big enough. +// Enlarge the biggest cached buffer, remove it from the cache and return it. +// This provide cached buffers size adjustment based on demand. +// +// In short, if the cache is not empty, Get guarantees to make it always one +// item less. This rules prevent uncontrolled cache grow in some scenarios. +// The older policy was not preventing that. Another advantage is better cached +// buffers sizes "auto tuning", although not in every possible use case. +// +// NOTE: The buffer returned by Get _is not guaranteed_ to be zeroed. That's +// okay for e.g. passing a buffer to io.Reader. If you need a zeroed buffer +// use Cget. +func (c *Cache) Get(n int) []byte { + r, _ := c.get(n) + return r +} + +func (c *Cache) get(n int) (r []byte, isZeroed bool) { + s := *c + lens := len(s) + if lens == 0 { + r, isZeroed = make([]byte, n, overCommit(n)), true + return + } + + i := sort.Search(lens, func(x int) bool { return len(s[x]) >= n }) + if i == lens { + i-- + s[i] = make([]byte, n, overCommit(n)) + } + r = s[i][:n] + copy(s[i:], s[i+1:]) + s[lens-1] = nil + s = s[:lens-1] + *c = s + return r, false +} + +// Cget will acquire a buffer using Get and then clears it to zeros. The +// zeroing goes up to n, not cap(r). +func (c *Cache) Cget(n int) (r []byte) { + r, ok := c.get(n) + if ok { + return + } + + for i := range r { + r[i] = 0 + } + return +} + +// Put caches b for possible later reuse (via Get). No other references to b's +// backing array may exist. Otherwise a big mess is sooner or later inevitable. +func (c *Cache) Put(b []byte) { + b = b[:cap(b)] + lenb := len(b) + if lenb == 0 { + return + } + + s := *c + lens := len(s) + i := sort.Search(lens, func(x int) bool { return len(s[x]) >= lenb }) + s = append(s, nil) + copy(s[i+1:], s[i:]) + s[i] = b + *c = s + return +} + +// Stats reports memory consumed by a Cache, without accounting for some +// (smallish) additional overhead. 'n' is the number of cached buffers, bytes +// is their combined capacity. +func (c Cache) Stats() (n, bytes int) { + n = len(c) + for _, v := range c { + bytes += cap(v) + } + return +} + +// CCache is a Cache which is safe for concurrent use by multiple goroutines. +type CCache struct { + c Cache + mu sync.Mutex +} + +// Get returns a buffer ([]byte) of length n. If no such buffer is cached then +// a biggest cached buffer is resized to have length n and returned. If there +// are no cached items at all, Get returns a newly allocated buffer. +// +// In other words the cache policy is: +// +// - If the cache is empty, the buffer must be newly created and returned. +// Cache remains empty. +// +// - If a buffer of sufficient size is found in the cache, remove it from the +// cache and return it. +// +// - Otherwise the cache is non empty, but no cached buffer is big enough. +// Enlarge the biggest cached buffer, remove it from the cache and return it. +// This provide cached buffers size adjustment based on demand. +// +// In short, if the cache is not empty, Get guarantees to make it always one +// item less. This rules prevent uncontrolled cache grow in some scenarios. +// The older policy was not preventing that. Another advantage is better cached +// buffers sizes "auto tuning", although not in every possible use case. +// +// NOTE: The buffer returned by Get _is not guaranteed_ to be zeroed. That's +// okay for e.g. passing a buffer to io.Reader. If you need a zeroed buffer +// use Cget. +func (c *CCache) Get(n int) []byte { + c.mu.Lock() + r, _ := c.c.get(n) + c.mu.Unlock() + return r +} + +// Cget will acquire a buffer using Get and then clears it to zeros. The +// zeroing goes up to n, not cap(r). +func (c *CCache) Cget(n int) (r []byte) { + c.mu.Lock() + r = c.c.Cget(n) + c.mu.Unlock() + return +} + +// Put caches b for possible later reuse (via Get). No other references to b's +// backing array may exist. Otherwise a big mess is sooner or later inevitable. +func (c *CCache) Put(b []byte) { + c.mu.Lock() + c.c.Put(b) + c.mu.Unlock() +} + +// Stats reports memory consumed by a Cache, without accounting for some +// (smallish) additional overhead. 'n' is the number of cached buffers, bytes +// is their combined capacity. +func (c *CCache) Stats() (n, bytes int) { + c.mu.Lock() + n, bytes = c.c.Stats() + c.mu.Unlock() + return +} + +// GCache is a ready to use global instance of a CCache. +var GCache CCache + +func overCommit(n int) int { + switch { + case n < 8: + return 8 + case n < 1e5: + return 2 * n + case n < 1e6: + return 3 * n / 2 + default: + return n + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/bufs/bufs_test.go b/Godeps/_workspace/src/github.com/cznic/bufs/bufs_test.go new file mode 100644 index 00000000000..62400b00620 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/bufs/bufs_test.go @@ -0,0 +1,174 @@ +// Copyright 2014 The bufs Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bufs + +import ( + "fmt" + "path" + "runtime" + "testing" +) + +var dbg = func(s string, va ...interface{}) { + _, fn, fl, _ := runtime.Caller(1) + fmt.Printf("%s:%d: ", path.Base(fn), fl) + fmt.Printf(s, va...) + fmt.Println() +} + +func Test0(t *testing.T) { + b := New(0) + defer func() { + recover() + }() + + b.Alloc(1) + t.Fatal("unexpected success") +} + +func Test1(t *testing.T) { + b := New(1) + expected := false + defer func() { + if e := recover(); e != nil && !expected { + t.Fatal(fmt.Errorf("%v", e)) + } + }() + + b.Alloc(1) + expected = true + b.Alloc(1) + t.Fatal("unexpected success") +} + +func Test2(t *testing.T) { + b := New(1) + expected := false + defer func() { + if e := recover(); e != nil && !expected { + t.Fatal(fmt.Errorf("%v", e)) + } + }() + + b.Alloc(1) + b.Free() + b.Alloc(1) + expected = true + b.Alloc(1) + t.Fatal("unexpected success") +} + +func Test3(t *testing.T) { + b := New(1) + expected := false + defer func() { + if e := recover(); e != nil && !expected { + t.Fatal(fmt.Errorf("%v", e)) + } + }() + + b.Alloc(1) + b.Free() + expected = true + b.Free() + t.Fatal("unexpected success") +} + +const ( + N = 1e5 + bufSize = 1 << 12 +) + +type Foo struct { + result []byte +} + +func NewFoo() *Foo { + return &Foo{} +} + +func (f *Foo) Bar(n int) { + buf := make([]byte, n) + sum := 0 + for _, v := range buf { + sum += int(v) + } + f.result = append(f.result, byte(sum)) + f.Qux(n) +} + +func (f *Foo) Qux(n int) { + buf := make([]byte, n) + sum := 0 + for _, v := range buf { + sum += int(v) + } + f.result = append(f.result, byte(sum)) +} + +type FooBufs struct { + buffers Buffers + result []byte +} + +const maxFooDepth = 2 + +func NewFooBufs() *FooBufs { + return &FooBufs{buffers: New(maxFooDepth)} +} + +func (f *FooBufs) Bar(n int) { + buf := f.buffers.Alloc(n) + defer f.buffers.Free() + + sum := 0 + for _, v := range buf { + sum += int(v) + } + f.result = append(f.result, byte(sum)) + f.Qux(n) +} + +func (f *FooBufs) Qux(n int) { + buf := f.buffers.Alloc(n) + defer f.buffers.Free() + + sum := 0 + for _, v := range buf { + sum += int(v) + } + f.result = append(f.result, byte(sum)) +} + +func TestFoo(t *testing.T) { + foo := NewFoo() + for i := 0; i < N; i++ { + foo.Bar(bufSize) + } +} + +func TestFooBufs(t *testing.T) { + foo := NewFooBufs() + for i := 0; i < N; i++ { + foo.Bar(bufSize) + } + t.Log("buffers.Stats()", foo.buffers.Stats()) +} + +func BenchmarkFoo(b *testing.B) { + b.SetBytes(2 * bufSize) + foo := NewFoo() + for i := 0; i < b.N; i++ { + foo.Bar(bufSize) + } +} + +func BenchmarkFooBufs(b *testing.B) { + b.SetBytes(2 * bufSize) + foo := NewFooBufs() + for i := 0; i < b.N; i++ { + foo.Bar(bufSize) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc.go new file mode 100644 index 00000000000..887604def2c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc.go @@ -0,0 +1,324 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Two Phase Commit & Structural ACID + +package lldb + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "os" + + "github.com/cznic/fileutil" + "github.com/cznic/mathutil" +) + +var _ Filer = &ACIDFiler0{} // Ensure ACIDFiler0 is a Filer + +type acidWrite struct { + b []byte + off int64 +} + +type acidWriter0 ACIDFiler0 + +func (a *acidWriter0) WriteAt(b []byte, off int64) (n int, err error) { + f := (*ACIDFiler0)(a) + if f.bwal == nil { // new epoch + f.data = f.data[:0] + f.bwal = bufio.NewWriter(f.wal) + if err = a.writePacket([]interface{}{wpt00Header, walTypeACIDFiler0, ""}); err != nil { + return + } + } + + if err = a.writePacket([]interface{}{wpt00WriteData, b, off}); err != nil { + return + } + + f.data = append(f.data, acidWrite{b, off}) + return len(b), nil +} + +func (a *acidWriter0) writePacket(items []interface{}) (err error) { + f := (*ACIDFiler0)(a) + b, err := EncodeScalars(items...) + if err != nil { + return + } + + var b4 [4]byte + binary.BigEndian.PutUint32(b4[:], uint32(len(b))) + if _, err = f.bwal.Write(b4[:]); err != nil { + return + } + + if _, err = f.bwal.Write(b); err != nil { + return + } + + if m := (4 + len(b)) % 16; m != 0 { + var pad [15]byte + _, err = f.bwal.Write(pad[:16-m]) + } + return +} + +// WAL Packet Tags +const ( + wpt00Header = iota + wpt00WriteData + wpt00Checkpoint +) + +const ( + walTypeACIDFiler0 = iota +) + +// ACIDFiler0 is a very simple, synchronous implementation of 2PC. It uses a +// single write ahead log file to provide the structural atomicity +// (BeginUpdate/EndUpdate/Rollback) and durability (DB can be recovered from +// WAL if a crash occurred). +// +// ACIDFiler0 is a Filer. +// +// NOTE: Durable synchronous 2PC involves three fsyncs in this implementation +// (WAL, DB, zero truncated WAL). Where possible, it's recommended to collect +// transactions for, say one second before performing the two phase commit as +// the typical performance for rotational hard disks is about few tens of +// fsyncs per second atmost. For an example of such collective transaction +// approach please see the colecting FSM STT in Dbm's documentation[1]. +// +// [1]: http://godoc.org/github.com/cznic/exp/dbm +type ACIDFiler0 struct { + *RollbackFiler + wal *os.File + bwal *bufio.Writer + data []acidWrite + testHook bool // keeps WAL untruncated (once) + peakWal int64 // tracks WAL maximum used size + peakBitFilerPages int // track maximum transaction memory +} + +// NewACIDFiler0 returns a newly created ACIDFiler0 with WAL in wal. +// +// If the WAL is zero sized then a previous clean shutdown of db is taken for +// granted and no recovery procedure is taken. +// +// If the WAL is of non zero size then it is checked for having a +// commited/fully finished transaction not yet been reflected in db. If such +// transaction exists it's committed to db. If the recovery process finishes +// successfully, the WAL is truncated to zero size and fsync'ed prior to return +// from NewACIDFiler0. +func NewACIDFiler(db Filer, wal *os.File) (r *ACIDFiler0, err error) { + fi, err := wal.Stat() + if err != nil { + return + } + + r = &ACIDFiler0{wal: wal} + + if fi.Size() != 0 { + if err = r.recoverDb(db); err != nil { + return + } + } + + acidWriter := (*acidWriter0)(r) + + if r.RollbackFiler, err = NewRollbackFiler( + db, + func(sz int64) (err error) { + // Checkpoint + if err = acidWriter.writePacket([]interface{}{wpt00Checkpoint, sz}); err != nil { + return + } + + if err = r.bwal.Flush(); err != nil { + return + } + + r.bwal = nil + + if err = r.wal.Sync(); err != nil { + return + } + + wfi, err := r.wal.Stat() + switch err != nil { + case true: + // unexpected, but ignored + case false: + r.peakWal = mathutil.MaxInt64(wfi.Size(), r.peakWal) + } + + // Phase 1 commit complete + + for _, v := range r.data { + if _, err := db.WriteAt(v.b, v.off); err != nil { + return err + } + } + + if err = db.Truncate(sz); err != nil { + return + } + + if err = db.Sync(); err != nil { + return + } + + // Phase 2 commit complete + + if !r.testHook { + if err = r.wal.Truncate(0); err != nil { + return + } + + if _, err = r.wal.Seek(0, 0); err != nil { + return + } + } + + r.testHook = false + return r.wal.Sync() + + }, + acidWriter, + ); err != nil { + return + } + + return r, nil +} + +// PeakWALSize reports the maximum size WAL has ever used. +func (a ACIDFiler0) PeakWALSize() int64 { + return a.peakWal +} + +func (a *ACIDFiler0) readPacket(f *bufio.Reader) (items []interface{}, err error) { + var b4 [4]byte + n, err := io.ReadAtLeast(f, b4[:], 4) + if n != 4 { + return + } + + ln := int(binary.BigEndian.Uint32(b4[:])) + m := (4 + ln) % 16 + padd := (16 - m) % 16 + b := make([]byte, ln+padd) + if n, err = io.ReadAtLeast(f, b, len(b)); n != len(b) { + return + } + + return DecodeScalars(b[:ln]) +} + +func (a *ACIDFiler0) recoverDb(db Filer) (err error) { + fi, err := a.wal.Stat() + if err != nil { + return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: err} + } + + if sz := fi.Size(); sz%16 != 0 { + return &ErrILSEQ{Type: ErrFileSize, Name: a.wal.Name(), Arg: sz} + } + + f := bufio.NewReader(a.wal) + items, err := a.readPacket(f) + if err != nil { + return + } + + if len(items) != 3 || items[0] != int64(wpt00Header) || items[1] != int64(walTypeACIDFiler0) { + return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("invalid packet items %#v", items)} + } + + tr := NewBTree(nil) + + for { + items, err = a.readPacket(f) + if err != nil { + return + } + + if len(items) < 2 { + return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("too few packet items %#v", items)} + } + + switch items[0] { + case int64(wpt00WriteData): + if len(items) != 3 { + return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("invalid data packet items %#v", items)} + } + + b, off := items[1].([]byte), items[2].(int64) + var key [8]byte + binary.BigEndian.PutUint64(key[:], uint64(off)) + if err = tr.Set(key[:], b); err != nil { + return + } + case int64(wpt00Checkpoint): + var b1 [1]byte + if n, err := f.Read(b1[:]); n != 0 || err == nil { + return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("checkpoint n %d, err %v", n, err)} + } + + if len(items) != 2 { + return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("checkpoint packet invalid items %#v", items)} + } + + sz := items[1].(int64) + enum, err := tr.seekFirst() + if err != nil { + return err + } + + for { + k, v, err := enum.current() + if err != nil { + if fileutil.IsEOF(err) { + break + } + + return err + } + + if _, err = db.WriteAt(v, int64(binary.BigEndian.Uint64(k))); err != nil { + return err + } + + if err = enum.next(); err != nil { + if fileutil.IsEOF(err) { + break + } + + return err + } + } + + if err = db.Truncate(sz); err != nil { + return err + } + + if err = db.Sync(); err != nil { + return err + } + + // Recovery complete + + if err = a.wal.Truncate(0); err != nil { + return err + } + + return a.wal.Sync() + default: + return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("packet tag %v", items[0])} + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_docs.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_docs.go new file mode 100644 index 00000000000..02c993b8ba4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_docs.go @@ -0,0 +1,44 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Anatomy of a WAL file + +WAL file + A sequence of packets + +WAL packet, parts in slice notation + [0:4], 4 bytes: N uint32 // network byte order + [4:4+N], N bytes: payload []byte // gb encoded scalars + +Packets, including the 4 byte 'size' prefix, MUST BE padded to size == 0 (mod +16). The values of the padding bytes MUST BE zero. + +Encoded scalars first item is a packet type number (packet tag). The meaning of +any other item(s) of the payload depends on the packet tag. + +Packet definitions + + {wpt00Header int, typ int, s string} + typ: Must be zero (ACIDFiler0 file). + s: Any comment string, empty string is okay. + + This packet must be present only once - as the first packet of + a WAL file. + + {wpt00WriteData int, b []byte, off int64} + Write data (WriteAt(b, off)). + + {wpt00Checkpoint int, sz int64} + Checkpoint (Truncate(sz)). + + This packet must be present only once - as the last packet of + a WAL file. + +*/ + +package lldb + +//TODO optimize bitfiler/wal/2pc data above final size diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_test.go new file mode 100644 index 00000000000..e683b8647b4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_test.go @@ -0,0 +1,285 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Two Phase Commit & Structural ACID + +package lldb + +import ( + "bytes" + "encoding/binary" + "io/ioutil" + "math/rand" + "os" + "testing" + + "github.com/cznic/mathutil" +) + +var _ Filer = &truncFiler{} + +type truncFiler struct { + f Filer + fake *MemFiler + totalWritten int // Including silently dropped + realWritten int + limit int // -1: unlimited, n: silently stop writing after limit bytes +} + +func NewTruncFiler(f Filer, limit int) *truncFiler { + return &truncFiler{f: f, fake: NewMemFiler(), limit: limit} +} + +func (f *truncFiler) BeginUpdate() error { panic("internal error") } +func (f *truncFiler) Close() error { return f.f.Close() } +func (f *truncFiler) EndUpdate() error { panic("internal error") } +func (f *truncFiler) Name() string { return f.f.Name() } +func (f *truncFiler) PunchHole(off, sz int64) error { panic("internal error") } +func (f *truncFiler) ReadAt(b []byte, off int64) (int, error) { return f.fake.ReadAt(b, off) } +func (f *truncFiler) Rollback() error { panic("internal error") } +func (f *truncFiler) Size() (int64, error) { return f.fake.Size() } +func (f *truncFiler) Sync() error { return f.f.Sync() } + +func (f *truncFiler) Truncate(sz int64) error { + f.fake.Truncate(sz) + return f.f.Truncate(sz) +} + +func (f *truncFiler) WriteAt(b []byte, off int64) (n int, err error) { + rq := len(b) + n = f.totalWritten + if lim := f.limit; lim >= 0 && n+rq > lim { + over := n + rq - lim + rq -= over + rq = mathutil.Max(rq, 0) + } + + if n, err = f.fake.WriteAt(b, off); err != nil { + return + } + + f.totalWritten += n + if rq != 0 { + n, err := f.f.WriteAt(b[:rq], off) + if err != nil { + return n, err + } + f.realWritten += n + } + return +} + +// Verify memory BTrees don't have maxRq limits. +func TestACID0MemBTreeCaps(t *testing.T) { + rng := rand.New(rand.NewSource(42)) + tr := NewBTree(nil) + b := make([]byte, 2*maxRq) + for i := range b { + b[i] = byte(rng.Int()) + } + + if err := tr.Set(nil, b); err != nil { + t.Fatal(len(b), err) + } + + g, err := tr.Get(nil, nil) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(g, b) { + t.Fatal("data mismatach") + } +} + +func TestACIDFiler0(t *testing.T) { + const SZ = 1 << 17 + + // Phase 1: Create a DB, fill with it with data. + + wal, err := ioutil.TempFile("", "test-acidfiler0-wal-") + if err != nil { + t.Fatal(err) + } + + if !*oKeep { + defer os.Remove(wal.Name()) + } + + db, err := ioutil.TempFile("", "test-acidfiler0-db-") + if err != nil { + t.Fatal(err) + } + + dbName := db.Name() + if !*oKeep { + defer os.Remove(db.Name()) + } + + realFiler := NewSimpleFileFiler(db) + truncFiler := NewTruncFiler(realFiler, -1) + acidFiler, err := NewACIDFiler(truncFiler, wal) + if err != nil { + t.Error(err) + return + } + + if err = acidFiler.BeginUpdate(); err != nil { + t.Error(err) + return + } + + a, err := NewAllocator(acidFiler, &Options{}) + if err != nil { + t.Error(err) + return + } + + a.Compress = true + + tr, h, err := CreateBTree(a, nil) + if h != 1 || err != nil { + t.Error(h, err) + return + } + + rng := rand.New(rand.NewSource(42)) + var key, val [8]byte + ref := map[int64]int64{} + + for { + sz, err := acidFiler.Size() + if err != nil { + t.Error(err) + return + } + + if sz > SZ { + break + } + + k, v := rng.Int63(), rng.Int63() + ref[k] = v + binary.BigEndian.PutUint64(key[:], uint64(k)) + binary.BigEndian.PutUint64(val[:], uint64(v)) + if err := tr.Set(key[:], val[:]); err != nil { + t.Error(err) + return + } + } + + acidFiler.testHook = true // keep WAL + + if err := acidFiler.EndUpdate(); err != nil { + t.Error(err) + return + } + + if err := acidFiler.Close(); err != nil { + t.Error(err) + return + } + + if err := wal.Sync(); err != nil { + t.Error(err) + return + } + + if _, err = wal.Seek(0, 0); err != nil { + t.Error(err) + return + } + + // Phase 2: Reopen and verify structure and data. + db, err = os.OpenFile(dbName, os.O_RDWR, 0666) + if err != nil { + t.Error(err) + return + } + + filer := NewSimpleFileFiler(db) + a, err = NewAllocator(filer, &Options{}) + if err != nil { + t.Error(err) + return + } + + if err = a.Verify(NewMemFiler(), nil, nil); err != nil { + t.Error(err) + return + } + + tr, err = OpenBTree(a, nil, 1) + for k, v := range ref { + binary.BigEndian.PutUint64(key[:], uint64(k)) + binary.BigEndian.PutUint64(val[:], uint64(v)) + var b []byte + b, err = tr.Get(b, key[:]) + if err != nil || b == nil || !bytes.Equal(b, val[:]) { + t.Error(err, b, val[:]) + return + } + } + + okImage, err := ioutil.ReadFile(dbName) + if err != nil { + t.Error(err) + return + } + + // Phase 3: Simulate a crash + sz, err := filer.Size() + if err != nil { + t.Error(err) + return + } + + sz /= 2 + if err := db.Truncate(sz); err != nil { + t.Error(err) + return + } + + z := make([]byte, sz/3) + n, err := db.WriteAt(z, sz/3) + if n != len(z) { + t.Error(n, err) + return + } + + if err := db.Sync(); err != nil { + t.Error(err) + return + } + + // Phase 4: Open the corrupted DB + filer = NewSimpleFileFiler(db) + acidFiler, err = NewACIDFiler(filer, wal) + if err != nil { + t.Error(err) + return + } + + if err = acidFiler.Sync(); err != nil { + t.Error(err) + return + } + + if err = acidFiler.Close(); err != nil { + t.Error(err) + return + } + + // Phase 5: Verify DB was recovered. + newImage, err := ioutil.ReadFile(dbName) + if err != nil { + t.Error(err) + return + } + + if !bytes.Equal(okImage, newImage) { + t.Error(err) + return + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/AUTHORS b/Godeps/_workspace/src/github.com/cznic/exp/lldb/AUTHORS new file mode 100644 index 00000000000..0078f5f5b6d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/AUTHORS @@ -0,0 +1,11 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/exp/lldb/CONTRIBUTORS new file mode 100644 index 00000000000..5e86f0607ce --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/CONTRIBUTORS @@ -0,0 +1,9 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/LICENSE b/Godeps/_workspace/src/github.com/cznic/exp/lldb/LICENSE new file mode 100644 index 00000000000..27e4447a49f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The lldb Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/Makefile b/Godeps/_workspace/src/github.com/cznic/exp/lldb/Makefile new file mode 100644 index 00000000000..cd8248a8e5c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/Makefile @@ -0,0 +1,45 @@ +# Copyright 2014 The lldb Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +.PHONY: all editor clean cover nuke + +testbin=lldb.test +grep=--include=*.go + +all: editor + go build + go vet + golint . + go install + make todo + +clean: + go clean + rm -f *~ cov cov.html bad-dump good-dump lldb.test old.txt new.txt \ + test-acidfiler0-* _test.db _wal + +cover: + t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t + +editor: + go fmt + go test -i + go test -timeout 1h + +mem: + go test -c + ./$(testbin) -test.bench . -test.memprofile mem.out -test.memprofilerate 1 -test.timeout 24h + go tool pprof --lines --web --alloc_space $(testbin) mem.out + +nuke: clean + go clean -i + +todo: + @grep -nr $(grep) BUG * || true + @grep -nr $(grep) LATER * || true + @grep -nr $(grep) MAYBE * || true + @grep -nr $(grep) TODO * || true + @grep -nr $(grep) FIXME * || true + @grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * || true + @grep -nr $(grep) println * || true diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/README.md b/Godeps/_workspace/src/github.com/cznic/exp/lldb/README.md new file mode 100644 index 00000000000..470bf541d0c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/README.md @@ -0,0 +1,8 @@ +lldb +==== + +Package lldb (WIP) implements a low level database engine. + +Installation: $ go get github.com/cznic/exp/lldb + +Documentation: [godoc.org/github.com/cznic/exp/lldb](http://godoc.org/github.com/cznic/exp/lldb) diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/all_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/all_test.go new file mode 100644 index 00000000000..451302de3a0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/all_test.go @@ -0,0 +1,43 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "encoding/hex" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "time" +) + +const ( + testDbName = "_test.db" + walName = "_wal" +) + +func now() time.Time { return time.Now() } + +func hdump(b []byte) string { + return hex.Dump(b) +} + +func die() { + os.Exit(1) +} + +func stack() string { + buf := make([]byte, 1<<16) + return string(buf[:runtime.Stack(buf, false)]) +} + +func temp() (dir, name string) { + dir, err := ioutil.TempDir("", "test-lldb-") + if err != nil { + panic(err) + } + + return dir, filepath.Join(dir, "test.tmp") +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/btree.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/btree.go new file mode 100644 index 00000000000..a80df1088a4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/btree.go @@ -0,0 +1,2276 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "bytes" + "errors" + "fmt" + "io" + "sort" + "strings" + + "github.com/cznic/bufs" + "github.com/cznic/fileutil" + "github.com/cznic/sortutil" +) + +const ( + kData = 256 // [1, 512] + kIndex = 256 // [2, 2048] + kKV = 19 // Size of the key/value field in btreeDataPage + kSz = kKV - 1 - 7 // Content prefix size + kH = kKV - 7 // Content field offset for handle + tagBTreeDataPage = 1 + tagBTreeIndexPage = 0 +) + +// BTree is a B+tree[1][2], i.e. a variant which speeds up +// enumeration/iteration of the BTree. According to its origin it can be +// volatile (backed only by memory) or non-volatile (backed by a non-volatile +// Allocator). +// +// The specific implementation of BTrees in this package are B+trees with +// delayed split/concatenation (discussed in e.g. [3]). +// +// Note: No BTree methods returns io.EOF for physical Filer reads/writes. The +// io.EOF is returned only by bTreeEnumerator methods to indicate "no more K-V +// pair". +// +// [1]: http://en.wikipedia.org/wiki/B+tree +// [2]: http://zgking.com:8080/home/donghui/publications/books/dshandbook_BTree.pdf +// [3]: http://people.cs.aau.dk/~simas/aalg06/UbiquitBtree.pdf +type BTree struct { + store btreeStore + root btree + collate func(a, b []byte) int + serial uint64 +} + +// NewBTree returns a new, memory-only BTree. +func NewBTree(collate func(a, b []byte) int) *BTree { + store := newMemBTreeStore() + root, err := newBTree(store) + if err != nil { // should not happen + panic(err.Error()) + } + + return &BTree{store, root, collate, 0} +} + +// IsMem reports if t is a memory only BTree. +func (t *BTree) IsMem() (r bool) { + _, r = t.store.(*memBTreeStore) + return +} + +// Clear empties the tree. +func (t *BTree) Clear() (err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + t.serial++ + return t.root.clear(t.store) +} + +// Delete deletes key and its associated value from the tree. +func (t *BTree) Delete(key []byte) (err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + t.serial++ + _, err = t.root.extract(t.store, nil, t.collate, key) + return +} + +// DeleteAny deletes one key and its associated value from the tree. If the +// tree is empty on return then empty is true. +func (t *BTree) DeleteAny() (empty bool, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + t.serial++ + return t.root.deleteAny(t.store) +} + +func elem(v interface{}) string { + switch x := v.(type) { + default: + panic("internal error") + case nil: + return "nil" + case bool: + if x { + return "true" + } + + return "false" + case int64: + return fmt.Sprint(x) + case uint64: + return fmt.Sprint(x) + case float64: + s := fmt.Sprintf("%g", x) + if !strings.Contains(s, ".") { + s += "." + } + return s + case complex128: + s := fmt.Sprint(x) + return s[1 : len(s)-1] + case []byte: + return fmt.Sprintf("[]byte{% 02x}", x) + case string: + return fmt.Sprintf("%q", x) + } +} + +// Dump outputs a human readable dump of t to w. It is usable iff t keys and +// values are encoded scalars (see EncodeScalars). Intended use is only for +// examples or debugging. Some type information is lost in the rendering, for +// example a float value '17.' and an integer value '17' may both output as +// '17'. +func (t *BTree) Dump(w io.Writer) (err error) { + enum, err := t.seekFirst() + if err != nil { + return + } + + for { + bkey, bval, err := enum.current() + if err != nil { + return err + } + + key, err := DecodeScalars(bkey) + if err != nil { + return err + } + + val, err := DecodeScalars(bval) + if err != nil { + return err + } + + kk := []string{} + if key == nil { + kk = []string{"null"} + } + for _, v := range key { + kk = append(kk, elem(v)) + } + vv := []string{} + if val == nil { + vv = []string{"null"} + } + for _, v := range val { + vv = append(vv, elem(v)) + } + skey := strings.Join(kk, ", ") + sval := strings.Join(vv, ", ") + if len(vv) > 1 { + sval = fmt.Sprintf("[]interface{%s}", sval) + } + if _, err = fmt.Fprintf(w, "%s → %s\n", skey, sval); err != nil { + return err + } + + err = enum.next() + if err != nil { + if fileutil.IsEOF(err) { + err = nil + break + } + + return err + } + } + return +} + +// Extract is a combination of Get and Delete. If the key exists in the tree, +// it is returned (like Get) and also deleted from a tree in a more efficient +// way which doesn't walk it twice. The returned slice may be a sub-slice of +// buf if buf was large enough to hold the entire content. Otherwise, a newly +// allocated slice will be returned. It is valid to pass a nil buf. +func (t *BTree) Extract(buf, key []byte) (value []byte, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + t.serial++ + return t.root.extract(t.store, buf, t.collate, key) +} + +// First returns the first KV pair of the tree, if it exists. Otherwise key == nil +// and value == nil. +func (t *BTree) First() (key, value []byte, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + var p btreeDataPage + if _, p, err = t.root.first(t.store); err != nil || p == nil { + return + } + + if key, err = p.key(t.store, 0); err != nil { + return + } + + value, err = p.value(t.store, 0) + return +} + +// Get returns the value associated with key, or nil if no such value exists. +// The returned slice may be a sub-slice of buf if buf was large enough to hold +// the entire content. Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil buf. +func (t *BTree) Get(buf, key []byte) (value []byte, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + buffer := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(buffer) + if buffer, err = t.root.get(t.store, buffer, t.collate, key); buffer == nil || err != nil { + return + } + + value = need(len(buffer), buf) + copy(value, buffer) + return +} + +// Handle reports t's handle. +func (t *BTree) Handle() int64 { + return int64(t.root) +} + +// Last returns the last KV pair of the tree, if it exists. Otherwise key == nil +// and value == nil. +func (t *BTree) Last() (key, value []byte, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + var p btreeDataPage + if _, p, err = t.root.last(t.store); err != nil || p == nil { + return + } + + index := p.len() - 1 + if key, err = p.key(t.store, index); err != nil { + return + } + + value, err = p.value(t.store, index) + return +} + +// Put combines Get and Set in a more efficient way where the tree is walked +// only once. The upd(ater) receives the current (key, old-value), if that +// exists or (key, nil) otherwise. It can then return a (new-value, true, nil) +// to create or overwrite the existing value in the KV pair, or (whatever, +// false, nil) if it decides not to create or not to update the value of the KV +// pair. +// +// tree.Set(k, v) +// +// conceptually equals +// +// tree.Put(k, func(k, v []byte){ return v, true }([]byte, bool)) +// +// modulo the differing return values. +// +// The returned slice may be a sub-slice of buf if buf was large enough to hold +// the entire content. Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil buf. +func (t *BTree) Put(buf, key []byte, upd func(key, old []byte) (new []byte, write bool, err error)) (old []byte, written bool, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + t.serial++ + return t.root.put2(buf, t.store, t.collate, key, upd) +} + +// Seek returns an Enumerator with "position" or an error of any. Normally the +// position is on a KV pair such that key >= KV.key. Then hit is key == KV.key. +// The position is possibly "after" the last KV pair, but that is not an error. +func (t *BTree) Seek(key []byte) (enum *BTreeEnumerator, hit bool, err error) { + enum0, hit, err := t.seek(key) + if err != nil { + return + } + + enum = &BTreeEnumerator{ + enum: enum0, + firstHit: hit, + key: append([]byte(nil), key...), + } + return +} + +func (t *BTree) seek(key []byte) (enum *bTreeEnumerator, hit bool, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + r := &bTreeEnumerator{t: t, collate: t.collate, serial: t.serial} + if r.p, r.index, hit, err = t.root.seek(t.store, r.collate, key); err != nil { + return + } + + enum = r + return +} + +// IndexSeek returns an Enumerator with "position" or an error of any. Normally +// the position is on a KV pair such that key >= KV.key. Then hit is key == +// KV.key. The position is possibly "after" the last KV pair, but that is not +// an error. The collate function originally passed to CreateBTree is used for +// enumerating the tree but a custom collate function c is used for IndexSeek. +func (t *BTree) IndexSeek(key []byte, c func(a, b []byte) int) (enum *BTreeEnumerator, hit bool, err error) { //TODO +test + enum0, hit, err := t.indexSeek(key, c) + if err != nil { + return + } + + enum = &BTreeEnumerator{ + enum: enum0, + firstHit: hit, + key: append([]byte(nil), key...), + } + return +} + +func (t *BTree) indexSeek(key []byte, c func(a, b []byte) int) (enum *bTreeEnumerator, hit bool, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + r := &bTreeEnumerator{t: t, collate: t.collate, serial: t.serial} + if r.p, r.index, hit, err = t.root.seek(t.store, c, key); err != nil { + return + } + + enum = r + return +} + +// seekFirst returns an enumerator positioned on the first KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returend. +func (t *BTree) SeekFirst() (enum *BTreeEnumerator, err error) { + enum0, err := t.seekFirst() + if err != nil { + return + } + + var key []byte + if key, _, err = enum0.current(); err != nil { + return + } + + enum = &BTreeEnumerator{ + enum: enum0, + firstHit: true, + key: append([]byte(nil), key...), + } + return +} + +func (t *BTree) seekFirst() (enum *bTreeEnumerator, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + var p btreeDataPage + if _, p, err = t.root.first(t.store); err == nil && p == nil { + err = io.EOF + } + if err != nil { + return + } + + return &bTreeEnumerator{t: t, collate: t.collate, p: p, index: 0, serial: t.serial}, nil +} + +// seekLast returns an enumerator positioned on the last KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returend. +func (t *BTree) SeekLast() (enum *BTreeEnumerator, err error) { + enum0, err := t.seekLast() + if err != nil { + return + } + + var key []byte + if key, _, err = enum0.current(); err != nil { + return + } + + enum = &BTreeEnumerator{ + enum: enum0, + firstHit: true, + key: append([]byte(nil), key...), + } + return +} + +func (t *BTree) seekLast() (enum *bTreeEnumerator, err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + var p btreeDataPage + if _, p, err = t.root.last(t.store); err == nil && p == nil { + err = io.EOF + } + if err != nil { + return + } + + return &bTreeEnumerator{t: t, collate: t.collate, p: p, index: p.len() - 1, serial: t.serial}, nil +} + +// Set sets the value associated with key. Any previous value, if existed, is +// overwritten by the new one. +func (t *BTree) Set(key, value []byte) (err error) { + if t == nil { + err = errors.New("BTree method invoked on nil receiver") + return + } + + t.serial++ + dst := bufs.GCache.Get(maxBuf) + _, err = t.root.put(dst, t.store, t.collate, key, value, true) + bufs.GCache.Put(dst) + return +} + +// bTreeEnumerator is a closure of a BTree and a position. It is returned from +// BTree.seek. +// +// NOTE: bTreeEnumerator cannot be used after its BTree was mutated after the +// bTreeEnumerator was acquired from any of the seek, seekFirst, seekLast +// methods. +type bTreeEnumerator struct { + t *BTree + collate func(a, b []byte) int + p btreeDataPage + index int + serial uint64 +} + +// Current returns the KV pair the enumerator is currently positioned on. If +// the position is before the first KV pair in the tree or after the last KV +// pair in the tree then err == io.EOF is returned. +// +// If the enumerator has been invalidated by updating the tree, ErrINVAL is +// returned. +func (e *bTreeEnumerator) current() (key, value []byte, err error) { + if e == nil { + err = errors.New("bTreeEnumerator method invoked on nil receiver") + return + } + + if e.serial != e.t.serial { + err = &ErrINVAL{Src: "bTreeEnumerator invalidated by updating the tree"} + return + } + + if e.p == nil || e.index == e.p.len() { + return nil, nil, io.EOF + } + + if key, err = e.p.key(e.t.store, e.index); err != nil { + return + } + + value, err = e.p.value(e.t.store, e.index) + return +} + +// Next attempts to position the enumerator onto the next KV pair wrt the +// current position. If there is no "next" KV pair, io.EOF is returned. +// +// If the enumerator has been invalidated by updating the tree, ErrINVAL is +// returned. +func (e *bTreeEnumerator) next() (err error) { + if e == nil { + err = errors.New("bTreeEnumerator method invoked on nil receiver") + return + } + + if e.serial != e.t.serial { + err = &ErrINVAL{Src: "bTreeEnumerator invalidated by updating the tree"} + return + } + + if e.p == nil { + return io.EOF + } + + switch { + case e.index < e.p.len()-1: + e.index++ + default: + ph := e.p.next() + if ph == 0 { + err = io.EOF + break + } + + if e.p, err = e.t.store.Get(e.p, ph); err != nil { + e.p = nil + return + } + e.index = 0 + } + return +} + +// Prev attempts to position the enumerator onto the previous KV pair wrt the +// current position. If there is no "previous" KV pair, io.EOF is returned. +// +// If the enumerator has been invalidated by updating the tree, ErrINVAL is +// returned. +func (e *bTreeEnumerator) prev() (err error) { + if e == nil { + err = errors.New("bTreeEnumerator method invoked on nil receiver") + return + } + + if e.serial != e.t.serial { + err = &ErrINVAL{Src: "bTreeEnumerator invalidated by updating the tree"} + return + } + + if e.p == nil { + return io.EOF + } + + switch { + case e.index > 0: + e.index-- + default: + ph := e.p.prev() + if ph == 0 { + err = io.EOF + break + } + + if e.p, err = e.t.store.Get(e.p, ph); err != nil { + e.p = nil + return + } + e.index = e.p.len() - 1 + } + return +} + +// BTreeEnumerator captures the state of enumerating a tree. It is returned +// from the Seek* methods. The enumerator is aware of any mutations made to +// the tree in the process of enumerating it and automatically resumes the +// enumeration. +type BTreeEnumerator struct { + enum *bTreeEnumerator + err error + key []byte + firstHit bool +} + +// Next returns the currently enumerated KV pair, if it exists and moves to the +// next KV in the key collation order. If there is no KV pair to return, err == +// io.EOF is returned. +func (e *BTreeEnumerator) Next() (key, value []byte, err error) { + if err = e.err; err != nil { + return + } + + canRetry := true +retry: + if key, value, err = e.enum.current(); err != nil { + if _, ok := err.(*ErrINVAL); !ok || !canRetry { + e.err = err + return + } + + canRetry = false + var hit bool + if e.enum, hit, err = e.enum.t.seek(e.key); err != nil { + e.err = err + return + } + + if !e.firstHit && hit { + err = e.enum.next() + if err != nil { + e.err = err + return + } + } + + goto retry + } + + e.firstHit = false + e.key = append([]byte(nil), key...) + e.err = e.enum.next() + return +} + +// Prev returns the currently enumerated KV pair, if it exists and moves to the +// previous KV in the key collation order. If there is no KV pair to return, +// err == io.EOF is returned. +func (e *BTreeEnumerator) Prev() (key, value []byte, err error) { + if err = e.err; err != nil { + return + } + + canRetry := true +retry: + if key, value, err = e.enum.current(); err != nil { + if _, ok := err.(*ErrINVAL); !ok || !canRetry { + e.err = err + return + } + + canRetry = false + var hit bool + if e.enum, hit, err = e.enum.t.seek(e.key); err != nil { + e.err = err + return + } + + if !e.firstHit && hit { + err = e.enum.prev() + if err != nil { + e.err = err + return + } + } + + goto retry + } + + e.firstHit = false + e.key = append([]byte(nil), key...) + e.err = e.enum.prev() + return +} + +// CreateBTree creates a new BTree in store. It returns the tree, its (freshly +// assigned) handle (for OpenBTree or RemoveBTree) or an error, if any. +func CreateBTree(store *Allocator, collate func(a, b []byte) int) (bt *BTree, handle int64, err error) { + r := &BTree{store: store, collate: collate} + if r.root, err = newBTree(store); err != nil { + return + } + + return r, int64(r.root), nil +} + +// OpenBTree opens a store's BTree using handle. It returns the tree or an +// error, if any. The same tree may be opened more than once, but operations on +// the separate instances should not ever overlap or void the other instances. +// However, the intended API usage is to open the same tree handle only once +// (handled by some upper layer "dispatcher"). +func OpenBTree(store *Allocator, collate func(a, b []byte) int, handle int64) (bt *BTree, err error) { + r := &BTree{store: store, root: btree(handle), collate: collate} + b := bufs.GCache.Get(7) + defer bufs.GCache.Put(b) + if b, err = store.Get(b, handle); err != nil { + return + } + + if len(b) != 7 { + return nil, &ErrILSEQ{Off: h2off(handle), More: "btree.go:671"} + } + + return r, nil +} + +// RemoveBTree removes tree, represented by handle from store. Empty trees are +// cheap, each uses only few bytes of the store. If there's a chance that a +// tree will eventually get reused (non empty again), it's recommended to +// not/never remove it. One advantage of such approach is a stable handle of +// such tree. +func RemoveBTree(store *Allocator, handle int64) (err error) { + tree, err := OpenBTree(store, nil, handle) + if err != nil { + return + } + + if err = tree.Clear(); err != nil { + return + } + + return store.Free(handle) +} + +type btreeStore interface { + Alloc(b []byte) (handle int64, err error) + Free(handle int64) (err error) + Get(dst []byte, handle int64) (b []byte, err error) + Realloc(handle int64, b []byte) (err error) +} + +// Read only zero bytes +var zeros [2 * kKV]byte + +func init() { + if kData < 1 || kData > 512 { + panic(fmt.Errorf("kData %d: out of limits", kData)) + } + + if kIndex < 2 || kIndex > 2048 { + panic(fmt.Errorf("kIndex %d: out of limits", kIndex)) + } + + if kKV < 8 || kKV > 23 { + panic(fmt.Errorf("kKV %d: out of limits", kKV)) + } + + if n := len(zeros); n < 15 { + panic(fmt.Errorf("not enough zeros: %d", n)) + } +} + +type memBTreeStore struct { + h int64 + m map[int64][]byte +} + +func newMemBTreeStore() *memBTreeStore { + return &memBTreeStore{h: 0, m: map[int64][]byte{}} +} + +func (s *memBTreeStore) String() string { + var a sortutil.Int64Slice + for k := range s.m { + a = append(a, k) + } + sort.Sort(a) + var sa []string + for _, k := range a { + sa = append(sa, fmt.Sprintf("%#x:|% x|", k, s.m[k])) + } + return strings.Join(sa, "\n") +} + +func (s *memBTreeStore) Alloc(b []byte) (handle int64, err error) { + s.h++ + handle = s.h + s.m[handle] = bpack(b) + return +} + +func (s *memBTreeStore) Free(handle int64) (err error) { + if _, ok := s.m[handle]; !ok { + return &ErrILSEQ{Type: ErrOther, Off: h2off(handle), More: "btree.go:754"} + } + + delete(s.m, handle) + return +} + +func (s *memBTreeStore) Get(dst []byte, handle int64) (b []byte, err error) { + r, ok := s.m[handle] + if !ok { + return nil, &ErrILSEQ{Type: ErrOther, Off: h2off(handle), More: "btree.go:764"} + } + + b = need(len(r), dst) + copy(b, r) + return +} + +func (s *memBTreeStore) Realloc(handle int64, b []byte) (err error) { + if _, ok := s.m[handle]; !ok { + return &ErrILSEQ{Type: ErrOther, Off: h2off(handle), More: "btree.go:774"} + } + + s.m[handle] = bpack(b) + return +} + +/* + +0...0 (1 bytes): +Flag + + 0 + +---+ + | 0 | + +---+ + +0 indicates an index page + +1...count*14-1 +"array" of items, 14 bytes each. Count of items in kIndex-1..2*kIndex+2 + + Count = (len(raw) - 8) / 14 + + 0..6 7..13 + +-------+----------+ + | Child | DataPage | + +-------+----------+ + + Child == handle of a child index page + DataPage == handle of a data page + +Offsets into the raw []byte: +Child[X] == 1+14*X +DataPage[X] == 8+14*X + +*/ +type btreeIndexPage []byte + +func newBTreeIndexPage(leftmostChild int64) (p btreeIndexPage) { + p = bufs.GCache.Get(1 + (kIndex+1)*2*7)[:8] + p[0] = tagBTreeIndexPage + h2b(p[1:], leftmostChild) + return +} + +func (p btreeIndexPage) len() int { + return (len(p) - 8) / 14 +} + +func (p btreeIndexPage) child(index int) int64 { + return b2h(p[1+14*index:]) +} + +func (p btreeIndexPage) setChild(index int, dp int64) { + h2b(p[1+14*index:], dp) +} + +func (p btreeIndexPage) dataPage(index int) int64 { + return b2h(p[8+14*index:]) +} + +func (p btreeIndexPage) setDataPage(index int, dp int64) { + h2b(p[8+14*index:], dp) +} + +func (q btreeIndexPage) insert(index int) btreeIndexPage { + switch len0 := q.len(); { + case index < len0: + has := len(q) + need := has + 14 + switch { + case cap(q) >= need: + q = q[:need] + default: + q = append(q, zeros[:14]...) + } + copy(q[8+14*(index+1):8+14*(index+1)+2*(len0-index)*7], q[8+14*index:]) + case index == len0: + has := len(q) + need := has + 14 + switch { + case cap(q) >= need: + q = q[:need] + default: + q = append(q, zeros[:14]...) + } + } + return q +} + +func (p btreeIndexPage) insert3(index int, dataPage, child int64) btreeIndexPage { + p = p.insert(index) + p.setDataPage(index, dataPage) + p.setChild(index+1, child) + return p +} + +func (p btreeIndexPage) cmp(a btreeStore, c func(a, b []byte) int, keyA []byte, keyBIndex int) (int, error) { + b := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(b) + dp, err := a.Get(b, p.dataPage(keyBIndex)) + if err != nil { + return 0, err + } + + return btreeDataPage(dp).cmp(a, c, keyA, 0) +} + +func (q btreeIndexPage) setLen(n int) btreeIndexPage { + q = q[:cap(q)] + need := 8 + 14*n + if need < len(q) { + return q[:need] + } + return append(q, make([]byte, need-len(q))...) +} + +func (p btreeIndexPage) split(a btreeStore, root btree, ph *int64, parent int64, parentIndex int, index *int) (btreeIndexPage, error) { + right := newBTreeIndexPage(0) + canRecycle := true + defer func() { + if canRecycle { + bufs.GCache.Put(right) + } + }() + right = right.setLen(kIndex) + copy(right[1:1+(2*kIndex+1)*7], p[1+14*(kIndex+1):]) + p = p.setLen(kIndex) + if err := a.Realloc(*ph, p); err != nil { + return nil, err + } + + rh, err := a.Alloc(right) + if err != nil { + return nil, err + } + + if parentIndex >= 0 { + var pp btreeIndexPage = bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(pp) + if pp, err = a.Get(pp, parent); err != nil { + return nil, err + } + pp = pp.insert3(parentIndex, p.dataPage(kIndex), rh) + if err = a.Realloc(parent, pp); err != nil { + return nil, err + } + + } else { + nr := newBTreeIndexPage(*ph) + defer bufs.GCache.Put(nr) + nr = nr.insert3(0, p.dataPage(kIndex), rh) + nrh, err := a.Alloc(nr) + if err != nil { + return nil, err + } + + if err = a.Realloc(int64(root), h2b(make([]byte, 7), nrh)); err != nil { + return nil, err + } + } + if *index > kIndex { + p = right + canRecycle = false + *ph = rh + *index -= kIndex + 1 + } + return p, nil +} + +// p is dirty on return +func (p btreeIndexPage) extract(index int) btreeIndexPage { + n := p.len() - 1 + if index < n { + sz := (n-index)*14 + 7 + copy(p[1+14*index:1+14*index+sz], p[1+14*(index+1):]) + } + return p.setLen(n) +} + +// must persist all changes made +func (p btreeIndexPage) underflow(a btreeStore, root, iroot, parent int64, ph *int64, parentIndex int, index *int) (btreeIndexPage, error) { + lh, rh, err := checkSiblings(a, parent, parentIndex) + if err != nil { + return nil, err + } + + var left btreeIndexPage = bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(left) + + if lh != 0 { + if left, err = a.Get(left, lh); err != nil { + return nil, err + } + + if lc := btreeIndexPage(left).len(); lc > kIndex { + var pp = bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(pp) + if pp, err = a.Get(pp, parent); err != nil { + return nil, err + } + + pc := p.len() + p = p.setLen(pc + 1) + di, si, sz := 1+1*14, 1+0*14, (2*pc+1)*7 + copy(p[di:di+sz], p[si:]) + p.setChild(0, btreeIndexPage(left).child(lc)) + p.setDataPage(0, btreeIndexPage(pp).dataPage(parentIndex-1)) + *index++ + btreeIndexPage(pp).setDataPage(parentIndex-1, btreeIndexPage(left).dataPage(lc-1)) + left = left.setLen(lc - 1) + if err = a.Realloc(parent, pp); err != nil { + return nil, err + } + + if err = a.Realloc(*ph, p); err != nil { + return nil, err + } + + return p, a.Realloc(lh, left) + } + } + + if rh != 0 { + right := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(right) + if right, err = a.Get(right, rh); err != nil { + return nil, err + } + + if rc := btreeIndexPage(right).len(); rc > kIndex { + pp := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(pp) + if pp, err = a.Get(pp, parent); err != nil { + return nil, err + } + + pc := p.len() + p = p.setLen(pc + 1) + p.setDataPage(pc, btreeIndexPage(pp).dataPage(parentIndex)) + pc++ + p.setChild(pc, btreeIndexPage(right).child(0)) + btreeIndexPage(pp).setDataPage(parentIndex, btreeIndexPage(right).dataPage(0)) + di, si, sz := 1+0*14, 1+1*14, (2*rc+1)*7 + copy(right[di:di+sz], right[si:]) + right = btreeIndexPage(right).setLen(rc - 1) + if err = a.Realloc(parent, pp); err != nil { + return nil, err + } + + if err = a.Realloc(*ph, p); err != nil { + return nil, err + } + + return p, a.Realloc(rh, right) + } + } + + if lh != 0 { + *index += left.len() + 1 + if left, err = left.concat(a, root, iroot, parent, lh, *ph, parentIndex-1); err != nil { + return p, err + } + + p, *ph = left, lh + return p, nil + } + + return p.concat(a, root, iroot, parent, *ph, rh, parentIndex) +} + +// must persist all changes made +func (p btreeIndexPage) concat(a btreeStore, root, iroot, parent, ph, rh int64, parentIndex int) (btreeIndexPage, error) { + pp := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(pp) + pp, err := a.Get(pp, parent) + if err != nil { + return nil, err + } + + right := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(right) + if right, err = a.Get(right, rh); err != nil { + return nil, err + } + + pc := p.len() + rc := btreeIndexPage(right).len() + p = p.setLen(pc + rc + 1) + p.setDataPage(pc, btreeIndexPage(pp).dataPage(parentIndex)) + di, si, sz := 1+14*(pc+1), 1+0*14, (2*rc+1)*7 + copy(p[di:di+sz], right[si:]) + if err := a.Realloc(ph, p); err != nil { + return nil, err + } + + if err := a.Free(rh); err != nil { + return nil, err + } + + if pc := btreeIndexPage(pp).len(); pc > 1 { + if parentIndex < pc-1 { + di, si, sz := 8+parentIndex*14, 8+(parentIndex+1)*14, 2*(pc-1-parentIndex)*7 + copy(pp[di:si+sz], pp[si:]) + } + pp = btreeIndexPage(pp).setLen(pc - 1) + return p, a.Realloc(parent, pp) + } + + if err := a.Free(iroot); err != nil { + return nil, err + } + + b7 := bufs.GCache.Get(7) + defer bufs.GCache.Put(b7) + return p, a.Realloc(root, h2b(b7[:7], ph)) +} + +/* + +0...0 (1 bytes): +Flag + + 0 + +---+ + | 1 | + +---+ + +1 indicates a data page + +1...14 (14 bytes) + + 1..7 8..14 + +------+------+ + | Prev | Next | + +------+------+ + + Prev, Next == Handles of the data pages doubly linked list + + Count = (len(raw) - 15) / (2*kKV) + +15...count*2*kKV-1 +"array" of items, 2*kKV bytes each. Count of items in kData-1..2*kData + +Item + 0..kKV-1 kKV..2*kKV-1 + +----------+--------------+ + | Key | Value | + +----------+--------------+ + +Key/Value encoding + +Length 0...kKV-1 + + 0 1...N N+1...kKV-1 + +---+---------+-------------+ + | N | Data | Padding | + +---+---------+-------------+ + + N == content length + Data == Key or Value content + Padding == MUST be zero bytes + +Length >= kKV + + 0 1...kkV-8 kKV-7...kkV-1 + +------+-----------+--------------+ + | 0xFF | Data | H | + +------+-----------+--------------+ + + Data == Key or Value content, first kKV-7 bytes + H == Handle to THE REST of the content, w/o the first bytes in Data. + +Offsets into the raw []byte: +Key[X] == 15+2*kKV*X +Value[X] == 15+kKV+2*kKV*X +*/ +type btreeDataPage []byte + +func newBTreeDataPage() (p btreeDataPage) { + p = bufs.GCache.Cget(1 + 2*7 + (kData+1)*2*kKV)[:1+2*7] + p[0] = tagBTreeDataPage + return +} + +func newBTreeDataPageAlloc(a btreeStore) (p btreeDataPage, h int64, err error) { + p = newBTreeDataPage() + h, err = a.Alloc(p) + return +} + +func (p btreeDataPage) len() int { + return (len(p) - 15) / (2 * kKV) +} + +func (q btreeDataPage) setLen(n int) btreeDataPage { + q = q[:cap(q)] + need := 15 + 2*kKV*n + if need < len(q) { + return q[:need] + } + return append(q, make([]byte, need-len(q))...) +} + +func (p btreeDataPage) prev() int64 { + return b2h(p[1:]) +} + +func (p btreeDataPage) next() int64 { + return b2h(p[8:]) +} + +func (p btreeDataPage) setPrev(h int64) { + h2b(p[1:], h) +} + +func (p btreeDataPage) setNext(h int64) { + h2b(p[8:], h) +} + +func (q btreeDataPage) insert(index int) btreeDataPage { + switch len0 := q.len(); { + case index < len0: + has := len(q) + need := has + 2*kKV + switch { + case cap(q) >= need: + q = q[:need] + default: + q = append(q, zeros[:2*kKV]...) + } + q.copy(q, index+1, index, len0-index) + return q + case index == len0: + has := len(q) + need := has + 2*kKV + switch { + case cap(q) >= need: + return q[:need] + default: + return append(q, zeros[:2*kKV]...) + } + } + panic("internal error") +} + +func (p btreeDataPage) contentField(off int) (b []byte, h int64) { + p = p[off:] + switch n := int(p[0]); { + case n >= kKV: // content has a handle + b = append([]byte(nil), p[1:1+kSz]...) + h = b2h(p[kH:]) + default: // content is embedded + b, h = append([]byte(nil), p[1:1+n]...), 0 + } + return +} + +func (p btreeDataPage) content(a btreeStore, off int) (b []byte, err error) { + b, h := p.contentField(off) + if h == 0 { + return + } + + // content has a handle + b2, err := a.Get(nil, h) //TODO buffers: Later, not a public API + if err != nil { + return nil, err + } + + return append(b, b2...), nil +} + +func (p btreeDataPage) setContent(a btreeStore, off int, b []byte) (err error) { + p = p[off:] + switch { + case p[0] >= kKV: // existing content has a handle + switch n := len(b); { + case n < kKV: + p[0] = byte(n) + if err = a.Free(b2h(p[kH:])); err != nil { + return + } + copy(p[1:], b) + default: + // reuse handle + copy(p[1:1+kSz], b) + return a.Realloc(b2h(p[kH:]), b[kSz:]) + } + default: // existing content is embedded + switch n := len(b); { + case n < kKV: + p[0] = byte(n) + copy(p[1:], b) + default: + p[0] = 0xff + copy(p[1:1+kSz], b) + h, err := a.Alloc(b[kSz:]) + if err != nil { + return err + } + + h2b(p[kH:], h) + } + } + return +} + +func (p btreeDataPage) keyField(index int) (b []byte, h int64) { + return p.contentField(15 + 2*kKV*index) +} + +func (p btreeDataPage) key(a btreeStore, index int) (b []byte, err error) { + return p.content(a, 15+2*kKV*index) +} + +func (p btreeDataPage) valueField(index int) (b []byte, h int64) { + return p.contentField(15 + kKV + 2*kKV*index) +} + +func (p btreeDataPage) value(a btreeStore, index int) (b []byte, err error) { + return p.content(a, 15+kKV+2*kKV*index) +} + +func (p btreeDataPage) valueCopy(a btreeStore, index int) (b []byte, err error) { + if b, err = p.content(a, 15+kKV+2*kKV*index); err != nil { + return + } + + return append([]byte(nil), b...), nil +} + +func (p btreeDataPage) setKey(a btreeStore, index int, key []byte) (err error) { + return p.setContent(a, 15+2*kKV*index, key) +} + +func (p btreeDataPage) setValue(a btreeStore, index int, value []byte) (err error) { + return p.setContent(a, 15+kKV+2*kKV*index, value) +} + +func (p btreeDataPage) cmp(a btreeStore, c func(a, b []byte) int, keyA []byte, keyBIndex int) (y int, err error) { + var keyB []byte + if keyB, err = p.content(a, 15+2*kKV*keyBIndex); err != nil { + return + } + + return c(keyA, keyB), nil +} + +func (p btreeDataPage) copy(src btreeDataPage, di, si, n int) { + do, so := 15+2*kKV*di, 15+2*kKV*si + copy(p[do:do+2*kKV*n], src[so:]) +} + +// {p,left} dirty on exit +func (p btreeDataPage) moveLeft(left btreeDataPage, n int) (btreeDataPage, btreeDataPage) { + nl, np := left.len(), p.len() + left = left.setLen(nl + n) + left.copy(p, nl, 0, n) + p.copy(p, 0, n, np-n) + return p.setLen(np - n), left +} + +func (p btreeDataPage) moveRight(right btreeDataPage, n int) (btreeDataPage, btreeDataPage) { + nr, np := right.len(), p.len() + right = right.setLen(nr + n) + right.copy(right, n, 0, nr) + right.copy(p, 0, np-n, n) + return p.setLen(np - n), right +} + +func (p btreeDataPage) insertItem(a btreeStore, index int, key, value []byte) (btreeDataPage, error) { + p = p.insert(index) + di, sz := 15+2*kKV*index, 2*kKV + copy(p[di:di+sz], zeros[:sz]) + if err := p.setKey(a, index, key); err != nil { + return nil, err + } + return p, p.setValue(a, index, value) +} + +func (p btreeDataPage) split(a btreeStore, root, ph, parent int64, parentIndex, index int, key, value []byte) (btreeDataPage, error) { + right, rh, err := newBTreeDataPageAlloc(a) + // fails defer bufs.GCache.Put(right) + if err != nil { + return nil, err + } + + if next := p.next(); next != 0 { + right.setNext(p.next()) + nxh := right.next() + nx := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(nx) + if nx, err = a.Get(nx, nxh); err != nil { + return nil, err + } + + btreeDataPage(nx).setPrev(rh) + if err = a.Realloc(nxh, nx); err != nil { + return nil, err + } + } + + p.setNext(rh) + right.setPrev(ph) + right = right.setLen(kData) + right.copy(p, 0, kData, kData) + p = p.setLen(kData) + + if parentIndex >= 0 { + var pp btreeIndexPage = bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(pp) + if pp, err = a.Get(pp, parent); err != nil { + return nil, err + } + + pp = pp.insert3(parentIndex, rh, rh) + if err = a.Realloc(parent, pp); err != nil { + return nil, err + } + + } else { + nr := newBTreeIndexPage(ph) + defer bufs.GCache.Put(nr) + nr = nr.insert3(0, rh, rh) + nrh, err := a.Alloc(nr) + if err != nil { + return nil, err + } + + if err = a.Realloc(root, h2b(make([]byte, 7), nrh)); err != nil { + return nil, err + } + + } + if index > kData { + if right, err = right.insertItem(a, index-kData, key, value); err != nil { + return nil, err + } + } else { + if p, err = p.insertItem(a, index, key, value); err != nil { + return nil, err + } + } + if err = a.Realloc(ph, p); err != nil { + return nil, err + } + + return p, a.Realloc(rh, right) +} + +func (p btreeDataPage) overflow(a btreeStore, root, ph, parent int64, parentIndex, index int, key, value []byte) (btreeDataPage, error) { + leftH, rightH, err := checkSiblings(a, parent, parentIndex) + if err != nil { + return nil, err + } + + if leftH != 0 { + left := btreeDataPage(bufs.GCache.Get(maxBuf)) + defer bufs.GCache.Put(left) + if left, err = a.Get(left, leftH); err != nil { + return nil, err + } + + if left.len() < 2*kData { + + p, left = p.moveLeft(left, 1) + if err = a.Realloc(leftH, left); err != nil { + return nil, err + } + + if p, err = p.insertItem(a, index-1, key, value); err != nil { + return nil, err + } + + return p, a.Realloc(ph, p) + } + } + + if rightH != 0 { + right := btreeDataPage(bufs.GCache.Get(maxBuf)) + defer bufs.GCache.Put(right) + if right, err = a.Get(right, rightH); err != nil { + return nil, err + } + + if right.len() < 2*kData { + if index < 2*kData { + p, right = p.moveRight(right, 1) + if err = a.Realloc(rightH, right); err != nil { + return nil, err + } + + if p, err = p.insertItem(a, index, key, value); err != nil { + return nil, err + } + + return p, a.Realloc(ph, p) + } else { + if right, err = right.insertItem(a, 0, key, value); err != nil { + return nil, err + } + + return p, a.Realloc(rightH, right) + } + } + } + return p.split(a, root, ph, parent, parentIndex, index, key, value) +} + +func (p btreeDataPage) swap(a btreeStore, di int, value []byte, canOverwrite bool) (oldValue []byte, err error) { + if oldValue, err = p.value(a, di); err != nil { + return + } + + if !canOverwrite { + return + } + + oldValue = append([]byte(nil), oldValue...) + err = p.setValue(a, di, value) + return +} + +type btreePage []byte + +func (p btreePage) isIndex() bool { + return p[0] == tagBTreeIndexPage +} + +func (p btreePage) len() int { + if p.isIndex() { + return btreeIndexPage(p).len() + } + + return btreeDataPage(p).len() +} + +func (p btreePage) find(a btreeStore, c func(a, b []byte) int, key []byte) (index int, ok bool, err error) { + l := 0 + h := p.len() - 1 + isIndex := p.isIndex() + if c == nil { + c = bytes.Compare + } + for l <= h { + index = (l + h) >> 1 + var cmp int + if isIndex { + if cmp, err = btreeIndexPage(p).cmp(a, c, key, index); err != nil { + return + } + } else { + if cmp, err = btreeDataPage(p).cmp(a, c, key, index); err != nil { + return + } + } + switch ok = cmp == 0; { + case cmp > 0: + l = index + 1 + case ok: + return + default: + h = index - 1 + } + } + return l, false, nil +} + +// p is dirty after extract! +func (p btreeDataPage) extract(a btreeStore, index int) (btreeDataPage, []byte, error) { + value, err := p.valueCopy(a, index) + if err != nil { + return nil, nil, err + } + + if _, h := p.keyField(index); h != 0 { + if err = a.Free(h); err != nil { + return nil, nil, err + } + } + + if _, h := p.valueField(index); h != 0 { + if err = a.Free(h); err != nil { + return nil, nil, err + } + } + + n := p.len() - 1 + if index < n { + p.copy(p, index, index+1, n-index) + } + return p.setLen(n), value, nil +} + +func checkSiblings(a btreeStore, parent int64, parentIndex int) (left, right int64, err error) { + if parentIndex >= 0 { + var p btreeIndexPage = bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(p) + if p, err = a.Get(p, parent); err != nil { + return + } + + if parentIndex > 0 { + left = p.child(parentIndex - 1) + } + if parentIndex < p.len() { + right = p.child(parentIndex + 1) + } + } + return +} + +// underflow must persist all changes made. +func (p btreeDataPage) underflow(a btreeStore, root, iroot, parent, ph int64, parentIndex int) (err error) { + lh, rh, err := checkSiblings(a, parent, parentIndex) + if err != nil { + return err + } + + if lh != 0 { + left := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(left) + if left, err = a.Get(left, lh); err != nil { + return err + } + + if btreeDataPage(left).len()+p.len() >= 2*kData { + left, p = btreeDataPage(left).moveRight(p, 1) + if err = a.Realloc(lh, left); err != nil { + return err + } + + return a.Realloc(ph, p) + } + } + + if rh != 0 { + right := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(right) + if right, err = a.Get(right, rh); err != nil { + return err + } + + if p.len()+btreeDataPage(right).len() > 2*kData { + right, p = btreeDataPage(right).moveLeft(p, 1) + if err = a.Realloc(rh, right); err != nil { + return err + } + + return a.Realloc(ph, p) + } + } + + if lh != 0 { + left := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(left) + if left, err = a.Get(left, lh); err != nil { + return err + } + + if err = a.Realloc(ph, p); err != nil { + return err + } + + return btreeDataPage(left).concat(a, root, iroot, parent, lh, ph, parentIndex-1) + } + + return p.concat(a, root, iroot, parent, ph, rh, parentIndex) +} + +// concat must persist all changes made. +func (p btreeDataPage) concat(a btreeStore, root, iroot, parent, ph, rh int64, parentIndex int) (err error) { + right := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(right) + if right, err = a.Get(right, rh); err != nil { + return err + } + + right, p = btreeDataPage(right).moveLeft(p, btreeDataPage(right).len()) + nxh := btreeDataPage(right).next() + if nxh != 0 { + nx := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(nx) + if nx, err = a.Get(nx, nxh); err != nil { + return err + } + + btreeDataPage(nx).setPrev(ph) + if err = a.Realloc(nxh, nx); err != nil { + return err + } + } + p.setNext(nxh) + if err = a.Free(rh); err != nil { + return err + } + + pp := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(pp) + if pp, err = a.Get(pp, parent); err != nil { + return err + } + + if btreeIndexPage(pp).len() > 1 { + pp = btreeIndexPage(pp).extract(parentIndex) + btreeIndexPage(pp).setChild(parentIndex, ph) + if err = a.Realloc(parent, pp); err != nil { + return err + } + + return a.Realloc(ph, p) + } + + if err = a.Free(iroot); err != nil { + return err + } + + if err = a.Realloc(ph, p); err != nil { + return err + } + + var b7 [7]byte + return a.Realloc(root, h2b(b7[:], ph)) +} + +// external "root" is stable and contains the real root. +type btree int64 + +func newBTree(a btreeStore) (btree, error) { + r, err := a.Alloc(zeros[:7]) + return btree(r), err +} + +func (root btree) String(a btreeStore) string { + r := bufs.GCache.Get(16) + defer bufs.GCache.Put(r) + r, err := a.Get(r, int64(root)) + if err != nil { + panic(err) + } + + iroot := b2h(r) + m := map[int64]bool{int64(root): true} + + s := []string{fmt.Sprintf("tree %#x -> %#x\n====", root, iroot)} + if iroot == 0 { + return s[0] + } + + var f func(int64, string) + f = func(h int64, ind string) { + if m[h] { + return + } + + m[h] = true + var b btreePage = bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(b) + var err error + if b, err = a.Get(b, h); err != nil { + panic(err) + } + + s = append(s, fmt.Sprintf("%s@%#x", ind, h)) + switch b.isIndex() { + case true: + da := []int64{} + b := btreeIndexPage(b) + for i := 0; i < b.len(); i++ { + c, d := b.child(i), b.dataPage(i) + s = append(s, fmt.Sprintf("%schild[%d] %#x dataPage[%d] %#x", ind, i, c, i, d)) + da = append(da, c) + da = append(da, d) + } + i := b.len() + c := b.child(i) + s = append(s, fmt.Sprintf("%schild[%d] %#x", ind, i, c)) + for _, c := range da { + f(c, ind+" ") + } + f(c, ind+" ") + case false: + b := btreeDataPage(b) + s = append(s, fmt.Sprintf("%sprev %#x next %#x", ind, b.prev(), b.next())) + for i := 0; i < b.len(); i++ { + k, err := b.key(a, i) + if err != nil { + panic(err) + } + + v, err := b.value(a, i) + if err != nil { + panic(err) + } + + s = append(s, fmt.Sprintf("%sK[%d]|% x| V[%d]|% x|", ind, i, k, i, v)) + } + } + } + + f(int64(iroot), "") + return strings.Join(s, "\n") +} + +func (root btree) put(dst []byte, a btreeStore, c func(a, b []byte) int, key, value []byte, canOverwrite bool) (prev []byte, err error) { + prev, _, err = root.put2(dst, a, c, key, func(key, old []byte) (new []byte, write bool, err error) { + new, write = value, true + return + }) + return +} + +func (root btree) put2(dst []byte, a btreeStore, c func(a, b []byte) int, key []byte, upd func(key, old []byte) (new []byte, write bool, err error)) (old []byte, written bool, err error) { + var r, value []byte + if r, err = a.Get(dst, int64(root)); err != nil { + return + } + + iroot := b2h(r) + var h int64 + if iroot == 0 { + p := newBTreeDataPage() + defer bufs.GCache.Put(p) + if value, written, err = upd(key, nil); err != nil || !written { + return + } + + if p, err = p.insertItem(a, 0, key, value); err != nil { + return + } + + h, err = a.Alloc(p) + if err != nil { + return nil, true, err + } + + err = a.Realloc(int64(root), h2b(r, h)[:7]) + return + } + + parentIndex := -1 + var parent int64 + ph := iroot + + p := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(p) + + for { + if p, err = a.Get(p[:cap(p)], ph); err != nil { + return + } + + var index int + var ok bool + + if index, ok, err = btreePage(p).find(a, c, key); err != nil { + return + } + + switch { + case ok: // Key found + if btreePage(p).isIndex() { + ph = btreeIndexPage(p).dataPage(index) + if p, err = a.Get(p, ph); err != nil { + return + } + + if old, err = btreeDataPage(p).valueCopy(a, 0); err != nil { + return + } + + if value, written, err = upd(key, old); err != nil || !written { + return + } + + if _, err = btreeDataPage(p).swap(a, 0, value, true); err != nil { + return + } + + err = a.Realloc(ph, p) + return + } + + if old, err = btreeDataPage(p).valueCopy(a, index); err != nil { + return + } + + if value, written, err = upd(key, old); err != nil || !written { + return + } + + if _, err = btreeDataPage(p).swap(a, index, value, true); err != nil { + return + } + + err = a.Realloc(ph, p) + return + case btreePage(p).isIndex(): + if btreePage(p).len() > 2*kIndex { + if p, err = btreeIndexPage(p).split(a, root, &ph, parent, parentIndex, &index); err != nil { + return + } + } + parentIndex = index + parent = ph + ph = btreeIndexPage(p).child(index) + default: + if value, written, err = upd(key, nil); err != nil || !written { + return + } + + if btreePage(p).len() < 2*kData { // page is not full + if p, err = btreeDataPage(p).insertItem(a, index, key, value); err != nil { + return + } + + err = a.Realloc(ph, p) + return + } + + // page is full + p, err = btreeDataPage(p).overflow(a, int64(root), ph, parent, parentIndex, index, key, value) + return + } + } +} + +//TODO actually use 'dst' to return 'value' +func (root btree) get(a btreeStore, dst []byte, c func(a, b []byte) int, key []byte) (b []byte, err error) { + var r []byte + if r, err = a.Get(dst, int64(root)); err != nil { + return + } + + iroot := b2h(r) + if iroot == 0 { + return + } + + ph := iroot + + for { + var p btreePage + if p, err = a.Get(p, ph); err != nil { + return + } + + var index int + var ok bool + if index, ok, err = p.find(a, c, key); err != nil { + return + } + + switch { + case ok: + if p.isIndex() { + dh := btreeIndexPage(p).dataPage(index) + dp, err := a.Get(dst, dh) + if err != nil { + return nil, err + } + + return btreeDataPage(dp).value(a, 0) + } + + return btreeDataPage(p).value(a, index) + case p.isIndex(): + ph = btreeIndexPage(p).child(index) + default: + return + } + } +} + +//TODO actually use 'dst' to return 'value' +func (root btree) extract(a btreeStore, dst []byte, c func(a, b []byte) int, key []byte) (value []byte, err error) { + var r []byte + if r, err = a.Get(dst, int64(root)); err != nil { + return + } + + iroot := b2h(r) + if iroot == 0 { + return + } + + ph := iroot + parentIndex := -1 + var parent int64 + + p := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(p) + + for { + if p, err = a.Get(p[:cap(p)], ph); err != nil { + return + } + + var index int + var ok bool + if index, ok, err = btreePage(p).find(a, c, key); err != nil { + return + } + + if ok { + if btreePage(p).isIndex() { + dph := btreeIndexPage(p).dataPage(index) + dp, err := a.Get(dst, dph) + if err != nil { + return nil, err + } + + if btreeDataPage(dp).len() > kData { + if dp, value, err = btreeDataPage(dp).extract(a, 0); err != nil { + return nil, err + } + + return value, a.Realloc(dph, dp) + } + + if btreeIndexPage(p).len() < kIndex && ph != iroot { + var err error + if p, err = btreeIndexPage(p).underflow(a, int64(root), iroot, parent, &ph, parentIndex, &index); err != nil { + return nil, err + } + } + parentIndex = index + 1 + parent = ph + ph = btreeIndexPage(p).child(parentIndex) + continue + } + + p, value, err = btreeDataPage(p).extract(a, index) + if btreePage(p).len() >= kData { + err = a.Realloc(ph, p) + return + } + + if ph != iroot { + err = btreeDataPage(p).underflow(a, int64(root), iroot, parent, ph, parentIndex) + return + } + + if btreePage(p).len() == 0 { + if err = a.Free(ph); err != nil { + return + } + + err = a.Realloc(int64(root), zeros[:7]) + return + } + err = a.Realloc(ph, p) + return + } + + if !btreePage(p).isIndex() { + return + } + + if btreePage(p).len() < kIndex && ph != iroot { + if p, err = btreeIndexPage(p).underflow(a, int64(root), iroot, parent, &ph, parentIndex, &index); err != nil { + return nil, err + } + } + parentIndex = index + parent = ph + ph = btreeIndexPage(p).child(index) + } +} + +func (root btree) deleteAny(a btreeStore) (bool, error) { + r := bufs.GCache.Get(7) + defer bufs.GCache.Put(r) + var err error + if r, err = a.Get(r, int64(root)); err != nil { + return false, err + } + + iroot := b2h(r) + if iroot == 0 { + return true, nil + } + + ph := iroot + parentIndex := -1 + var parent int64 + p := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(p) + + for { + if p, err = a.Get(p, ph); err != nil { + return false, err + } + + index := btreePage(p).len() / 2 + if btreePage(p).isIndex() { + dph := btreeIndexPage(p).dataPage(index) + dp := bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(dp) + if dp, err = a.Get(dp, dph); err != nil { + return false, err + } + + if btreeDataPage(dp).len() > kData { + if dp, _, err = btreeDataPage(dp).extract(a, 0); err != nil { + return false, err + } + + return false, a.Realloc(dph, dp) + } + + if btreeIndexPage(p).len() < kIndex && ph != iroot { + if p, err = btreeIndexPage(p).underflow(a, int64(root), iroot, parent, &ph, parentIndex, &index); err != nil { + return false, err + } + } + parentIndex = index + 1 + parent = ph + ph = btreeIndexPage(p).child(parentIndex) + continue + } + + p, _, err = btreeDataPage(p).extract(a, index) + if btreePage(p).len() >= kData { + err = a.Realloc(ph, p) + return false, err + } + + if ph != iroot { + err = btreeDataPage(p).underflow(a, int64(root), iroot, parent, ph, parentIndex) + return false, err + } + + if btreePage(p).len() == 0 { + if err = a.Free(ph); err != nil { + return true, err + } + + return true, a.Realloc(int64(root), zeros[:7]) + } + + return false, a.Realloc(ph, p) + } +} + +func (root btree) first(a btreeStore) (ph int64, p btreeDataPage, err error) { + r := bufs.GCache.Get(7) + defer bufs.GCache.Put(r) + if r, err = a.Get(r, int64(root)); err != nil { + return + } + + for ph = b2h(r); ph != 0; ph = btreeIndexPage(p).child(0) { + if p, err = a.Get(p, ph); err != nil { + return + } + + if !btreePage(p).isIndex() { + break + } + } + + return +} + +func (root btree) last(a btreeStore) (ph int64, p btreeDataPage, err error) { + r := bufs.GCache.Get(7) + defer bufs.GCache.Put(r) + if r, err = a.Get(r, int64(root)); err != nil { + return + } + + for ph = b2h(r); ph != 0; ph = btreeIndexPage(p).child(btreeIndexPage(p).len()) { + if p, err = a.Get(p, ph); err != nil { + return + } + + if !btreePage(p).isIndex() { + break + } + } + + return +} + +// key >= p[index].key +func (root btree) seek(a btreeStore, c func(a, b []byte) int, key []byte) (p btreeDataPage, index int, equal bool, err error) { + r := bufs.GCache.Get(7) + defer bufs.GCache.Put(r) + if r, err = a.Get(r, int64(root)); err != nil { + return + } + + for ph := b2h(r); ph != 0; ph = btreeIndexPage(p).child(index) { + if p, err = a.Get(p, ph); err != nil { + break + } + + if index, equal, err = btreePage(p).find(a, c, key); err != nil { + break + } + + if equal { + if !btreePage(p).isIndex() { + break + } + + p, err = a.Get(p, btreeIndexPage(p).dataPage(index)) + index = 0 + break + } + + if !btreePage(p).isIndex() { + break + } + } + return +} + +func (root btree) clear(a btreeStore) (err error) { + r := bufs.GCache.Get(7) + defer bufs.GCache.Put(r) + if r, err = a.Get(r, int64(root)); err != nil { + return + } + + iroot := b2h(r) + if iroot == 0 { + return + } + + if err = root.clear2(a, iroot); err != nil { + return + } + + var b [7]byte + return a.Realloc(int64(root), b[:]) +} + +func (root btree) clear2(a btreeStore, ph int64) (err error) { + var p = bufs.GCache.Get(maxBuf) + defer bufs.GCache.Put(p) + if p, err = a.Get(p, ph); err != nil { + return + } + + switch btreePage(p).isIndex() { + case true: + ip := btreeIndexPage(p) + for i := 0; i <= ip.len(); i++ { + root.clear2(a, ip.child(i)) + + } + case false: + dp := btreeDataPage(p) + for i := 0; i < dp.len(); i++ { + if err = dp.setKey(a, i, nil); err != nil { + return + } + + if err = dp.setValue(a, i, nil); err != nil { + return + } + } + } + return a.Free(ph) +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/btree_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/btree_test.go new file mode 100644 index 00000000000..06c4ae042fc --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/btree_test.go @@ -0,0 +1,1887 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "flag" + "fmt" + "math" + "math/rand" + "os" + "runtime" + "testing" + + "github.com/cznic/fileutil" + "github.com/cznic/mathutil" +) + +var ( + testFrom = flag.Uint("from", 0, "test I [-from, -N)") + noGrow = flag.Bool("noGrow", false, "check only embeded keys/values") +) + +func verifyPageLinks(a btreeStore, tree btree, n int) (err error) { + var p btreeDataPage + var ph int64 + if ph, p, err = tree.first(a); err != nil { + return + } + + if n == 0 { + if ph != 0 || p != nil { + return fmt.Errorf("first() should returned nil page") + } + + ph2, p2, err := tree.last(a) + if err != nil { + return err + } + + if ph2 != 0 || p2 != nil { + return fmt.Errorf("last() should returned nil page") + } + + } + + n0 := n + var prev int64 + var lastKey []byte + for ph != 0 { + if p, err = a.Get(nil, ph); err != nil { + return + } + + if g, e := p.prev(), prev; g != e { + return fmt.Errorf("broken L-R DLL chain: p %d p.prev %d, exp %d", ph, g, e) + } + + for i := 0; i < p.len(); i++ { + key, err := p.key(a, i) + if err != nil { + return err + } + + if key == nil { + return fmt.Errorf("nil key") + } + + if lastKey != nil && !(bytes.Compare(lastKey, key) < 0) { + return fmt.Errorf("L-R key ordering broken") + } + + lastKey = key + n-- + } + + prev, ph = ph, p.next() + } + + if n != 0 { + return fmt.Errorf("# of keys off by %d (L-R)", n) + } + + n = n0 + if ph, p, err = tree.last(a); err != nil { + return + } + + lastKey = nil + var next int64 + + for ph != 0 { + if p, err = a.Get(nil, ph); err != nil { + return + } + + if g, e := p.next(), next; g != e { + return fmt.Errorf("broken R-L DLL chain") + } + + for i := p.len() - 1; i >= 0; i-- { + key, err := p.key(a, i) + if err != nil { + return err + } + + if key == nil { + return fmt.Errorf("nil key") + } + + if lastKey != nil && !(bytes.Compare(key, lastKey) < 0) { + return fmt.Errorf("R-L key ordering broken") + } + + lastKey = key + n-- + } + + next, ph = ph, p.prev() + } + + if n != 0 { + return fmt.Errorf("# of keys off by %d (R-L)", n) + } + + return +} + +func testBTreePut1(t *testing.T, nf func() btreeStore, grow, from, to, xor int64) (tree btree) { + if *noGrow { + grow = 0 + } + + a := nf() + if a == nil { + t.Fatal(a) + } + + var err error + tree, err = newBTree(a) + if err != nil { + t.Fatal(err) + } + + if err := verifyPageLinks(a, tree, 0); err != nil { + t.Fatal(err) + } + + // Write and read back + var k, v, prevValue [7]byte + + n := 0 + for i := from; i < to; i++ { + h2b(k[:], 0x0100000000+i^xor) + h2b(v[:], 0x0200000000+i^xor) + kk := append(make([]byte, grow*i), k[:]...) + vv := append(make([]byte, grow*i), v[:]...) + prev, err := tree.put(nil, a, nil, kk, vv, true) + if err != nil || len(prev) != 0 { + t.Fatal(i, prev, err) + } + + var buf []byte + if buf, err = tree.get(a, nil, nil, kk); err != nil { + t.Fatal(err) + } + + if !bytes.Equal(buf, vv) { + t.Fatalf("\nK %sG %sE %s%s", hex.Dump(kk), hex.Dump(buf), hex.Dump(vv), tree.String(a)) + } + + n++ + } + + if err := verifyPageLinks(a, tree, n); err != nil { + t.Fatalf("%s\n%s", err, tree.String(a)) + } + + // Overwrite, read and extract + for i := from; i < to; i++ { + h2b(k[:], 0x0100000000+i^xor) + h2b(prevValue[:], 0x0200000000+i^xor) + h2b(v[:], 0x0300000000+i^xor) + kk := append(make([]byte, grow*i), k[:]...) + vv := append(make([]byte, grow*i), v[:]...) + expPrev := append(make([]byte, grow*i), prevValue[:]...) + gotPrev, err := tree.put(nil, a, nil, kk, vv, true) + if err != nil { + t.Fatal(i, err) + } + + if !bytes.Equal(gotPrev, expPrev) { + t.Fatalf("\nK %sG %sE %s%s", hex.Dump(kk), hex.Dump(gotPrev), hex.Dump(expPrev), tree.String(a)) + } + + var buf []byte + if buf, err = tree.get(a, nil, nil, kk); err != nil { + t.Fatal(err) + } + + if !bytes.Equal(buf, vv) { + t.Fatalf("\n%s%s%s%s", hex.Dump(kk), hex.Dump(buf), hex.Dump(vv), tree.String(a)) + } + + buf = nil + if buf, err = tree.extract(a, nil, nil, kk); err != nil { + t.Fatal(err) + } + + if !bytes.Equal(buf, vv) { + t.Fatalf("i %d, from [%d, %d)\nK %sG %sE %s%s", i, from, to, hex.Dump(kk), hex.Dump(buf), hex.Dump(vv), tree.String(a)) + } + + buf = nil + if buf, err = tree.get(a, nil, nil, kk); err != nil { + t.Fatal(err) + } + + if buf != nil { + t.Fatalf("\nK %sB %s%s", hex.Dump(kk), hex.Dump(buf), tree.String(a)) + } + + buf = nil + if buf, err = tree.extract(a, nil, nil, kk); err != nil { + t.Fatal(err) + } + + if buf != nil { + t.Fatalf("\n%s\n%s%s", hex.Dump(kk), hex.Dump(buf), tree.String(a)) + } + + n-- + if err := verifyPageLinks(a, tree, n); err != nil { + t.Fatalf("%s\n%s", err, tree.String(a)) + } + } + + return +} + +var xors = [...]int64{0, 0xffffffff, 0x55555555, 0xaaaaaaaa} + +func TestBTreePutGetExtract(t *testing.T) { + N := int64(3 * kData) + from := int64(*testFrom) + + for grow := 0; grow < 2; grow++ { + for _, x := range xors { + var s *memBTreeStore + tree := testBTreePut1(t, func() btreeStore { s = newMemBTreeStore(); return s }, int64(grow), from, N, x) + if err := verifyPageLinks(s, tree, 0); err != nil { + t.Fatal(err) + } + + if g, e := len(s.m), 1; g != e { + t.Fatalf("leak(s) %d %d\n%s", g, e, s) + } + } + } +} + +func testBTreePut2(t *testing.T, nf func() btreeStore, grow, n int) (tree btree) { + if *noGrow { + grow = 0 + } + rng, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + t.Fatal(err) + } + + a := nf() + if a == nil { + t.Fatal(a) + } + + tree, err = newBTree(a) + if err != nil { + t.Fatal(err) + } + + var k, v [7]byte + for i := 0; i < n; i++ { + ik, iv := int64(rng.Next()), int64(rng.Next()) + h2b(k[:], ik) + h2b(v[:], iv) + kk := append(make([]byte, grow*i), k[:]...) + vv := append(make([]byte, grow*i), v[:]...) + prev, err := tree.put(nil, a, nil, kk, vv, true) + if err != nil || len(prev) != 0 { + t.Fatal(i, prev, err) + } + + var buf []byte + if buf, err = tree.get(a, nil, nil, kk); err != nil { + t.Fatal(err) + } + + if !bytes.Equal(buf, vv) { + t.Fatalf("\n%s%s%s%s", hex.Dump(kk), hex.Dump(buf), hex.Dump(vv), tree.String(a)) + } + } + + if err := verifyPageLinks(a, tree, n); err != nil { + t.Fatalf("%s\n%s\n", err, tree.String(a)) + } + + rng.Seek(0) + for i := 0; i < n; i++ { + ik, iv := int64(rng.Next()), int64(rng.Next()) + h2b(k[:], ik) + h2b(v[:], iv) + kk := append(make([]byte, grow*i), k[:]...) + vv := append(make([]byte, grow*i), v[:]...) + var buf []byte + buf, err := tree.extract(a, nil, nil, kk) + if err != nil { + t.Fatal(i, err) + } + + if !bytes.Equal(buf, vv) { + t.Fatalf("\n%s\n%s\n%s\n%s", hex.Dump(kk), hex.Dump(buf), hex.Dump(vv), tree.String(a)) + } + + if err := verifyPageLinks(a, tree, n-i-1); err != nil { + t.Fatalf("%s\n%s", err, tree.String(a)) + } + } + + return +} + +func TestBTreePutGetExtractRnd(t *testing.T) { + N := *testN + + for grow := 0; grow < 2; grow++ { + var s *memBTreeStore + tree := testBTreePut2(t, func() btreeStore { s = newMemBTreeStore(); return s }, grow, N) + if err := verifyPageLinks(s, tree, 0); err != nil { + t.Fatal(err) + } + + if g, e := len(s.m), 1; g != e { + t.Fatalf("leak(s) %d %d\n%s", g, e, s) + } + } +} + +func benchmarkBTreePut(b *testing.B, v []byte) { + b.StopTimer() + rng := rand.New(rand.NewSource(42)) + ka := make([][7]byte, b.N) + for _, v := range ka { + h2b(v[:], int64(rng.Int63())) + } + a := newMemBTreeStore() + tree, err := newBTree(a) + if err != nil { + b.Fatal(err) + } + + runtime.GC() + b.StartTimer() + for _, k := range ka { + tree.put(nil, a, bytes.Compare, k[:], v, true) + } +} + +func BenchmarkBTreePut1(b *testing.B) { + v := make([]byte, 1) + benchmarkBTreePut(b, v) +} + +func BenchmarkBTreePut8(b *testing.B) { + v := make([]byte, 8) + benchmarkBTreePut(b, v) +} + +func BenchmarkBTreePut16(b *testing.B) { + v := make([]byte, 16) + benchmarkBTreePut(b, v) +} + +func BenchmarkBTreePut32(b *testing.B) { + v := make([]byte, 32) + benchmarkBTreePut(b, v) +} + +func benchmarkBTreeGet(b *testing.B, v []byte) { + b.StopTimer() + rng := rand.New(rand.NewSource(42)) + ka := make([][7]byte, b.N) + for _, v := range ka { + h2b(v[:], int64(rng.Int63())) + } + a := newMemBTreeStore() + tree, err := newBTree(a) + if err != nil { + b.Fatal(err) + } + + for _, k := range ka { + tree.put(nil, a, bytes.Compare, k[:], v, true) + } + buf := make([]byte, len(v)) + runtime.GC() + b.StartTimer() + for _, k := range ka { + tree.get(a, buf, bytes.Compare, k[:]) + } +} + +func BenchmarkBTreeGet1(b *testing.B) { + v := make([]byte, 1) + benchmarkBTreeGet(b, v) +} + +func BenchmarkBTreeGet8(b *testing.B) { + v := make([]byte, 8) + benchmarkBTreeGet(b, v) +} + +func BenchmarkBTreeGet16(b *testing.B) { + v := make([]byte, 16) + benchmarkBTreeGet(b, v) +} + +func BenchmarkBTreeGet32(b *testing.B) { + v := make([]byte, 32) + benchmarkBTreeGet(b, v) +} + +func TestbTreeSeek(t *testing.T) { + N := int64(*testN) + + tree := NewBTree(nil) + + // Fill + for i := int64(1); i <= N; i++ { + tree.Set(enc8(10*i), enc8(10*i+1)) + } + + // Check + a, root := tree.store, tree.root + for i := int64(1); i <= N; i++ { + // Exact match + lowKey := enc8(10*i - 1) + key := enc8(10 * i) + highKey := enc8(10*i + 1) + p, index, eq, err := root.seek(a, nil, key) + if err != nil { + t.Fatal(err) + } + + if !eq { + t.Fatal(i) + } + + if btreePage(p).isIndex() { + t.Fatal(i, "need btreeDataPage") + } + + dp := btreeDataPage(p) + n := dp.len() + if n < 0 || n > 2*kData { + t.Fatal(i, n) + } + + if index < 0 || index >= n { + t.Fatal(index) + } + + g, err := dp.key(a, index) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(g, key) { + t.Fatal(i) + } + + g, err = dp.value(a, index) + if err != nil { + t.Fatal(err) + } + + value := enc8(10*i + 1) + if !bytes.Equal(g, value) { + t.Fatal(i) + } + + // Nonexistent "low" key. Search for 9 should return the key 10. + p, index, eq, err = root.seek(a, nil, lowKey) + if err != nil { + t.Fatal(err) + } + + if eq { + t.Fatal(i) + } + + if btreePage(p).isIndex() { + t.Fatal(i, "need btreeDataPage") + } + + dp = btreeDataPage(p) + n = dp.len() + if n < 0 || n > 2*kData { + t.Fatal(i, n) + } + + if index < 0 || index > n { + t.Fatal(index, n) + } + + if index == n { + ph := dp.next() + index = 0 + if dp, err = a.Get(p, ph); err != nil { + t.Fatal(err) + } + } + + g, err = dp.key(a, index) + if err != nil { + t.Fatal(err) + } + + expKey := key + if !bytes.Equal(g, expKey) { + fmt.Println(root.String(a)) + //t.Fatalf("%d low|% x| g|% x| e|% x|", i, lowKey, g, expKey) + } + + g, err = dp.value(a, index) + if err != nil { + t.Fatal(err) + } + + value = enc8(10*i + 1) + if !bytes.Equal(g, value) { + t.Fatal(i) + } + + // Nonexistent "high" key. Search for 11 should return the key 20. + p, index, eq, err = root.seek(a, nil, highKey) + if err != nil { + t.Fatal(err) + } + + if eq { + t.Fatal(i) + } + + if btreePage(p).isIndex() { + t.Fatal(i, "need btreeDataPage") + } + + dp = btreeDataPage(p) + n = dp.len() + if n < 0 || n > 2*kData { + t.Fatal(i, n) + } + + if index < 0 || index > n { + t.Fatal(index, n) + } + + if index == n { + ph := dp.next() + if i == N { + if ph != 0 { + t.Fatal(ph) + } + + continue + } + + index = 0 + if dp, err = a.Get(p, ph); err != nil { + t.Fatal(err) + } + } + + g, err = dp.key(a, index) + if err != nil { + t.Fatal(err) + } + + expKey = enc8(10 * (i + 1)) + if !bytes.Equal(g, expKey) { + //fmt.Println(root.String(a)) + t.Fatalf("%d low|% x| g|% x| e|% x|", i, lowKey, g, expKey) + } + + g, err = dp.value(a, index) + if err != nil { + t.Fatal(err) + } + + value = enc8(10*(i+1) + 1) + if !bytes.Equal(g, value) { + t.Fatal(i) + } + + } +} + +func enc8(n int64) []byte { + var b [8]byte + h2b(b[:], n) + return b[:] +} + +func dec8(b []byte) (int64, error) { + if len(b) != 0 { + return 0, fmt.Errorf("dec8: len != 8 but %d", len(b)) + } + + return b2h(b), nil +} + +func TestbTreeNext(t *testing.T) { + N := int64(*testN) + + tree := NewBTree(nil) + enum, _, err := tree.seek(enc8(0)) + if err != nil { + t.Fatal(err) + } + + if _, _, err = enum.current(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + if err = enum.next(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + if err = enum.prev(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + // Fill + for i := int64(1); i <= N; i++ { + tree.Set(enc8(10*i), enc8(10*i+1)) + } + + var eq bool + + enum, eq, err = tree.seek(enc8(0)) + if err != nil { + t.Fatal(err) + } + + if eq { + t.Fatal(eq) + } + + // index: 0 + if _, _, err = enum.current(); err != nil { + t.Fatal(err) + } + + if err = enum.next(); N > 1 && err != nil { + t.Fatal(err) + } + + enum, eq, err = tree.seek(enc8(N * 10)) + if err != nil { + t.Fatal(err) + } + + if !eq { + t.Fatal(eq) + } + + // index: N-1 + if _, _, err = enum.current(); err != nil { + t.Fatal(err) + } + + if err = enum.next(); N > 1 && !fileutil.IsEOF(err) { + t.Fatal(err) + } + + enum, eq, err = tree.seek(enc8(N*10 + 1)) + if err != nil { + t.Fatal(err) + } + + if eq { + t.Fatal(eq) + } + + // index: N + if _, _, err = enum.current(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + if err = enum.next(); N > 1 && !fileutil.IsEOF(err) { + t.Fatal(err) + } + + enum, _, err = tree.seek(enc8(0)) + if err != nil { + t.Fatal(err) + } + + for i := int64(1); i <= N; i++ { + expKey, expValue := enc8(10*i), enc8(10*i+1) + k, v, err := enum.current() + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(k, expKey) { + t.Fatal(i) + } + + if !bytes.Equal(v, expValue) { + t.Fatal(i) + } + + switch { + case i == N: + if err := enum.next(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + default: + if err := enum.next(); err != nil { + t.Fatal(err) + } + } + } +} + +func TestbTreePrev(t *testing.T) { + N := int64(*testN) + + tree := NewBTree(nil) + enum, _, err := tree.seek(enc8(0)) + if err != nil { + t.Fatal(err) + } + + if _, _, err = enum.current(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + if err = enum.next(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + if err = enum.prev(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + // Fill + for i := int64(1); i <= N; i++ { + tree.Set(enc8(10*i), enc8(10*i+1)) + } + + var eq bool + + enum, eq, err = tree.seek(enc8(0)) + if err != nil { + t.Fatal(err) + } + + if eq { + t.Fatal(eq) + } + + // index: 0 + if _, _, err = enum.current(); err != nil { + t.Fatal(err) + } + + if err = enum.prev(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + enum, eq, err = tree.seek(enc8(N * 10)) + if err != nil { + t.Fatal(err) + } + + if !eq { + t.Fatal(eq) + } + + // index: N-1 + if _, _, err = enum.current(); err != nil { + t.Fatal(err) + } + + if err = enum.prev(); N > 1 && err != nil { + t.Fatal(err) + } + + enum, eq, err = tree.seek(enc8(N*10 + 1)) + if err != nil { + t.Fatal(err) + } + + if eq { + t.Fatal(eq) + } + + // index: N + if _, _, err = enum.current(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + + if err = enum.prev(); err != nil { + t.Fatal(err) + } + + enum, _, err = tree.seek(enc8(N * 10)) + if err != nil { + t.Fatal(err) + } + + for i := N; i >= 1; i-- { + expKey, expValue := enc8(10*i), enc8(10*i+1) + k, v, err := enum.current() + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(k, expKey) { + t.Fatalf("%d k|% x| expK|% x| %s\n", i, k, expKey, tree.root.String(tree.store)) + } + + if !bytes.Equal(v, expValue) { + t.Fatal(i) + } + + switch { + case i == 1: + if err := enum.prev(); !fileutil.IsEOF(err) { + t.Fatal(err) + } + default: + if err := enum.prev(); err != nil { + t.Fatal(i, err) + } + } + } +} + +func TestBTreeClear(t *testing.T) { + N := int64(*testN) + + var err error + var p []byte + for n := int64(0); n <= N; n = n*3/2 + 1 { + tree := NewBTree(nil) + for i := int64(0); i < n; i++ { + k := append(make([]byte, kKV), enc8(10*i+1)...) + v := append(make([]byte, kKV+1), enc8(10*i+2)...) + if err = tree.Set(k, v); err != nil { + t.Fatal(err) + } + } + + if err = tree.Clear(); err != nil { + t.Fatal(err) + } + + if g, e := len(tree.store.(*memBTreeStore).m), 1; g != e { + t.Fatalf("%v %v %v\n%s", n, g, e, tree.store.(*memBTreeStore).String()) + } + + if p, err = tree.store.Get(p, 1); err != nil { + t.Fatal(err) + } + + if g, e := p, zeros[:7]; len(g) != 0 && !bytes.Equal(g, e) { + t.Fatalf("|% x| |% x|", g, e) + } + } +} + +func TestBTreeRemove(t *testing.T) { + N := int64(*testN) + + for n := int64(0); n <= N; n = n*3/2 + 1 { + f := NewMemFiler() + store, err := NewAllocator(f, &Options{}) + if err != nil { + t.Fatal(err) + } + + sz0, err := f.Size() + if err != nil { + t.Fatal(err) + } + + tree, handle, err := CreateBTree(store, nil) + if err != nil { + t.Fatal(err) + } + + for i := int64(0); i < n; i++ { + k := append(make([]byte, kKV), enc8(10*i+1)...) + v := append(make([]byte, kKV+1), enc8(10*i+2)...) + if err = tree.Set(k, v); err != nil { + t.Fatal(err) + } + } + + if err = RemoveBTree(store, handle); err != nil { + t.Fatal(err) + } + + sz, err := f.Size() + if err != nil { + t.Fatal(err) + } + + if g, e := sz-sz0, int64(0); g != e { + t.Fatal(g, e) + } + } +} + +func collate(a, b []byte) (r int) { + da, err := DecodeScalars(a) + if err != nil { + panic(err) + } + + db, err := DecodeScalars(b) + if err != nil { + panic(err) + } + + r, err = Collate(da, db, nil) + if err != nil { + panic(err) + } + + return +} + +func TestBTreeCollatingBug(t *testing.T) { + tree := NewBTree(collate) + + date, err := EncodeScalars("Date") + if err != nil { + t.Fatal(err) + } + + customer, err := EncodeScalars("Customer") + if err != nil { + t.Fatal(err) + } + + if g, e := collate(customer, date), -1; g != e { + t.Fatal(g, e) + } + + if g, e := collate(date, customer), 1; g != e { + t.Fatal(g, e) + } + + err = tree.Set(date, nil) + if err != nil { + t.Fatal(err) + } + + err = tree.Set(customer, nil) + if err != nil { + t.Fatal(err) + } + + var b bytes.Buffer + tree.Dump(&b) + t.Logf("\n%s", b.String()) + + key, _, err := tree.First() + if err != nil { + t.Fatal(err) + } + + if g, e := key, customer; !bytes.Equal(g, e) { + t.Fatal(g, e) + } + +} + +func TestExtract(t *testing.T) { // Test of the exported wrapper only, .extract tested elsewhere + bt := NewBTree(nil) + bt.Set([]byte("a"), []byte("b")) + bt.Set([]byte("c"), []byte("d")) + bt.Set([]byte("e"), []byte("f")) + + if v, err := bt.Get(nil, []byte("a")); string(v) != "b" || err != nil { + t.Fatal(v, err) + } + + if v, err := bt.Get(nil, []byte("c")); string(v) != "d" || err != nil { + t.Fatal(v, err) + } + + if v, err := bt.Get(nil, []byte("e")); string(v) != "f" || err != nil { + t.Fatal(v, err) + } + + if v, err := bt.Extract(nil, []byte("c")); string(v) != "d" || err != nil { + t.Fatal(v, err) + } + + if v, err := bt.Get(nil, []byte("a")); string(v) != "b" || err != nil { + t.Fatal(v, err) + } + + if v, err := bt.Get(nil, []byte("c")); v != nil || err != nil { + t.Fatal(v, err) + } + + if v, err := bt.Get(nil, []byte("e")); string(v) != "f" || err != nil { + t.Fatal(v, err) + } +} + +func TestFirst(t *testing.T) { + bt := NewBTree(nil) + + if k, v, err := bt.First(); k != nil || v != nil || err != nil { + t.Fatal(k, v, err) + } + + bt.Set([]byte("a"), []byte("b")) + bt.Set([]byte("c"), []byte("d")) + + if k, v, err := bt.First(); string(k) != "a" || string(v) != "b" || err != nil { + t.Fatal(k, v, err) + } + + if err := bt.Delete([]byte("a")); err != nil { + t.Fatal(err) + } + + if k, v, err := bt.First(); string(k) != "c" || string(v) != "d" || err != nil { + t.Fatal(k, v, err) + } + + if err := bt.Delete([]byte("c")); err != nil { + t.Fatal(err) + } + + if k, v, err := bt.First(); k != nil || v != nil || err != nil { + t.Fatal(k, v, err) + } +} + +func TestLast(t *testing.T) { + bt := NewBTree(nil) + + if k, v, err := bt.First(); k != nil || v != nil || err != nil { + t.Fatal(k, v, err) + } + + bt.Set([]byte("a"), []byte("b")) + bt.Set([]byte("c"), []byte("d")) + + if k, v, err := bt.Last(); string(k) != "c" || string(v) != "d" || err != nil { + t.Fatal(k, v, err) + } + + if err := bt.Delete([]byte("c")); err != nil { + t.Fatal(err) + } + + if k, v, err := bt.First(); string(k) != "a" || string(v) != "b" || err != nil { + t.Fatal(k, v, err) + } + + if err := bt.Delete([]byte("a")); err != nil { + t.Fatal(err) + } + + if k, v, err := bt.First(); k != nil || v != nil || err != nil { + t.Fatal(k, v, err) + } +} + +func TestseekFirst(t *testing.T) { + bt := NewBTree(nil) + + enum, err := bt.seekFirst() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + bt.Set([]byte("c"), []byte("d")) + enum, err = bt.seekFirst() + if err != nil { + t.Fatal(err) + } + + err = enum.prev() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + err = enum.next() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + k, v, err := enum.current() + if err != nil { + t.Fatal(err) + } + + if string(k) != "c" || string(v) != "d" { + t.Fatal(k, v) + } + + bt.Set([]byte("a"), []byte("b")) + enum, err = bt.seekFirst() + if err != nil { + t.Fatal(err) + } + + err = enum.prev() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + k, v, err = enum.current() + if err != nil { + t.Fatal(err) + } + + if string(k) != "a" || string(v) != "b" { + t.Fatal(k, v) + } + + err = enum.next() + if err != nil { + t.Fatal(err) + } + + k, v, err = enum.current() + if err != nil { + t.Fatal(err) + } + + if string(k) != "c" || string(v) != "d" { + t.Fatal(k, v) + } +} + +func TestseekLast(t *testing.T) { + bt := NewBTree(nil) + + enum, err := bt.seekFirst() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + bt.Set([]byte("a"), []byte("b")) + enum, err = bt.seekFirst() + if err != nil { + t.Fatal(err) + } + + err = enum.prev() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + err = enum.next() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + k, v, err := enum.current() + if err != nil { + t.Fatal(err) + } + + if string(k) != "a" || string(v) != "b" { + t.Fatal(k, v) + } + + bt.Set([]byte("c"), []byte("d")) + enum, err = bt.seekLast() + if err != nil { + t.Fatal(err) + } + + err = enum.next() + if !fileutil.IsEOF(err) { + t.Fatal(err) + } + + k, v, err = enum.current() + if err != nil { + t.Fatal(err) + } + + if string(k) != "c" || string(v) != "d" { + t.Fatal(k, v) + } + + err = enum.prev() + if err != nil { + t.Fatal(err) + } + + k, v, err = enum.current() + if err != nil { + t.Fatal(err) + } + + if string(k) != "a" || string(v) != "b" { + t.Fatal(k, v) + } +} + +func TestDeleteAny(t *testing.T) { + const N = 1e4 + rng := rand.New(rand.NewSource(42)) + ref := map[uint32]bool{} + tr := NewBTree(nil) + data := []byte{42} + var key [4]byte + for i := 0; i < N; i++ { + k := uint32(rng.Int()) + binary.LittleEndian.PutUint32(key[:], k) + if err := tr.Set(key[:], data); err != nil { + t.Fatal(err) + } + + ref[k] = true + } + + for i := len(ref); i != 0; i-- { + empty, err := tr.DeleteAny() + if err != nil { + t.Fatal(err) + } + + if empty && i != 1 { + t.Fatal(i) + } + } +} + +func benchmarkBTreeSetFiler(b *testing.B, f Filer, sz int) { + if err := f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + a, err := NewAllocator(f, &Options{}) + if err != nil { + b.Error(err) + return + } + + tr, _, err := CreateBTree(a, nil) + if err != nil { + f.EndUpdate() + b.Error(err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + + keys := make([][8]byte, b.N) + for i := range keys { + binary.BigEndian.PutUint64(keys[i][:], uint64(i)) + } + v := make([]byte, sz) + runtime.GC() + b.ResetTimer() + for _, k := range keys { + if err = f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + if err := tr.Set(k[:], v); err != nil { + f.EndUpdate() + b.Error(err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + } +} + +func benchmarkBTreeSetMemFiler(b *testing.B, sz int) { + f := NewMemFiler() + benchmarkBTreeSetFiler(b, f, sz) +} + +func BenchmarkBTreeSetMemFiler0(b *testing.B) { + benchmarkBTreeSetMemFiler(b, 0) +} + +func BenchmarkBTreeSetMemFiler1e1(b *testing.B) { + benchmarkBTreeSetMemFiler(b, 1e1) +} + +func BenchmarkBTreeSetMemFiler1e2(b *testing.B) { + benchmarkBTreeSetMemFiler(b, 1e2) +} + +func BenchmarkBTreeSetMemFiler1e3(b *testing.B) { + benchmarkBTreeSetMemFiler(b, 1e3) +} + +func benchmarkBTreeSetSimpleFileFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + benchmarkBTreeSetFiler(b, NewSimpleFileFiler(f), sz) +} + +func BenchmarkBTreeSetSimpleFileFiler0(b *testing.B) { + benchmarkBTreeSetSimpleFileFiler(b, 0) +} + +func BenchmarkBTreeSetSimpleFileFiler1e1(b *testing.B) { + benchmarkBTreeSetSimpleFileFiler(b, 1e1) +} + +func BenchmarkBTreeSetSimpleFileFiler1e2(b *testing.B) { + benchmarkBTreeSetSimpleFileFiler(b, 1e2) +} + +func BenchmarkBTreeSetSimpleFileFiler1e3(b *testing.B) { + benchmarkBTreeSetSimpleFileFiler(b, 1e3) +} + +func benchmarkBTreeSetRollbackFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + g := NewSimpleFileFiler(f) + var filer *RollbackFiler + if filer, err = NewRollbackFiler( + g, + func(sz int64) error { + if err = g.Truncate(sz); err != nil { + return err + } + + return g.Sync() + }, + g, + ); err != nil { + b.Error(err) + return + } + + benchmarkBTreeSetFiler(b, filer, sz) +} + +func BenchmarkBTreeSetRollbackFiler0(b *testing.B) { + benchmarkBTreeSetRollbackFiler(b, 0) +} + +func BenchmarkBTreeSetRollbackFiler1e1(b *testing.B) { + benchmarkBTreeSetRollbackFiler(b, 1e1) +} + +func BenchmarkBTreeSetRollbackFiler1e2(b *testing.B) { + benchmarkBTreeSetRollbackFiler(b, 1e2) +} + +func BenchmarkBTreeSetRollbackFiler1e3(b *testing.B) { + benchmarkBTreeSetRollbackFiler(b, 1e3) +} + +func benchmarkBTreeSetACIDFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + wal, err := os.OpenFile(testDbName+".wal", os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer wal.Close() + + filer, err := NewACIDFiler(NewSimpleFileFiler(f), wal) + if err != nil { + b.Error(err) + return + } + + benchmarkBTreeSetFiler(b, filer, sz) +} + +func BenchmarkBTreeSetACIDFiler0(b *testing.B) { + benchmarkBTreeSetACIDFiler(b, 0) +} + +func BenchmarkBTreeSetACIDFiler1e1(b *testing.B) { + benchmarkBTreeSetACIDFiler(b, 1e1) +} + +func BenchmarkBTreeSetACIDFiler1e2(b *testing.B) { + benchmarkBTreeSetACIDFiler(b, 1e2) +} + +func BenchmarkBTreeSetACIDFiler1e3(b *testing.B) { + benchmarkBTreeSetACIDFiler(b, 1e3) +} + +func testbTreeEnumeratorInvalidating(t *testing.T, mutate func(b *BTree) error) { + b := NewBTree(nil) + if err := b.Set([]byte{1}, []byte{2}); err != nil { + t.Fatal(err) + } + + if err := b.Set([]byte{3}, []byte{4}); err != nil { + t.Fatal(err) + } + + e, err := b.seekFirst() + if err != nil { + t.Fatal(err) + } + + if _, _, err = e.current(); err != nil { + t.Fatal(err) + } + + if err = e.next(); err != nil { + t.Fatal(err) + } + + if err = e.prev(); err != nil { + t.Fatal(err) + } + + if err = mutate(b); err != nil { + t.Fatal(err) + } + + if _, _, err = e.current(); err == nil { + t.Fatal(err) + } + + if _, ok := err.(*ErrINVAL); !ok { + t.Fatalf("%T", err) + } + + err = e.next() + if err == nil { + t.Fatal(err) + } + + if _, ok := err.(*ErrINVAL); !ok { + t.Fatalf("%T", err) + } + + err = e.prev() + if err == nil { + t.Fatal(err) + } + + if _, ok := err.(*ErrINVAL); !ok { + t.Fatalf("%T", err) + } + +} + +func TestBTreeEnumeratorInvalidating(t *testing.T) { + testbTreeEnumeratorInvalidating(t, func(b *BTree) error { return b.Clear() }) + testbTreeEnumeratorInvalidating(t, func(b *BTree) error { return b.Delete([]byte{1}) }) + testbTreeEnumeratorInvalidating(t, func(b *BTree) error { _, err := b.DeleteAny(); return err }) + testbTreeEnumeratorInvalidating(t, func(b *BTree) error { _, err := b.Extract(nil, []byte{1}); return err }) + testbTreeEnumeratorInvalidating(t, func(b *BTree) error { + _, _, err := b.Put( + nil, + []byte{1}, + func(k, o []byte) ([]byte, bool, error) { return nil, false, nil }, + ) + return err + }) + testbTreeEnumeratorInvalidating(t, func(b *BTree) error { return b.Set([]byte{4}, []byte{5}) }) +} + +func n2b(n int) []byte { + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(n)) + return b[:] +} + +func b2n(b []byte) int { + if len(b) != 8 { + return mathutil.MinInt + } + + return int(binary.BigEndian.Uint64(b)) +} + +func TestBTreeSeekNext(t *testing.T) { + // seeking within 3 keys: 10, 20, 30 + table := []struct { + k int + hit bool + keys []int + }{ + {5, false, []int{10, 20, 30}}, + {10, true, []int{10, 20, 30}}, + {15, false, []int{20, 30}}, + {20, true, []int{20, 30}}, + {25, false, []int{30}}, + {30, true, []int{30}}, + {35, false, []int{}}, + } + + for i, test := range table { + up := test.keys + db := NewBTree(nil) + + if err := db.Set(n2b(10), n2b(100)); err != nil { + t.Fatal(i, err) + } + + if err := db.Set(n2b(20), n2b(200)); err != nil { + t.Fatal(i, err) + } + + if err := db.Set(n2b(30), n2b(300)); err != nil { + t.Fatal(i, err) + } + + for brokenSerial := 0; brokenSerial < 16; brokenSerial++ { + en, hit, err := db.Seek(n2b(test.k)) + if err != nil { + t.Fatal(err) + } + + if g, e := hit, test.hit; g != e { + t.Fatal(i, g, e) + } + + j := 0 + for { + if brokenSerial&(1<= len(up) { + t.Fatal(i, j, brokenSerial) + } + + if g, e := b2n(k), up[j]; g != e { + t.Fatal(i, j, brokenSerial, g, e) + } + + if g, e := len(v), 8; g != e { + t.Fatal(i, g, e) + } + + if g, e := b2n(v), 10*up[j]; g != e { + t.Fatal(i, g, e) + } + + j++ + + } + + if g, e := j, len(up); g != e { + t.Fatal(i, j, g, e) + } + } + + } +} + +func TestBTreeSeekPrev(t *testing.T) { + // seeking within 3 keys: 10, 20, 30 + table := []struct { + k int + hit bool + keys []int + }{ + {5, false, []int{10}}, + {10, true, []int{10}}, + {15, false, []int{20, 10}}, + {20, true, []int{20, 10}}, + {25, false, []int{30, 20, 10}}, + {30, true, []int{30, 20, 10}}, + {35, false, []int{}}, + } + + for i, test := range table { + down := test.keys + db := NewBTree(nil) + if err := db.Set(n2b(10), n2b(100)); err != nil { + t.Fatal(i, err) + } + + if err := db.Set(n2b(20), n2b(200)); err != nil { + t.Fatal(i, err) + } + + if err := db.Set(n2b(30), n2b(300)); err != nil { + t.Fatal(i, err) + } + + for brokenSerial := 0; brokenSerial < 16; brokenSerial++ { + en, hit, err := db.Seek(n2b(test.k)) + if err != nil { + t.Fatal(err) + } + + if g, e := hit, test.hit; g != e { + t.Fatal(i, g, e) + } + + j := 0 + for { + if brokenSerial&(1<= len(down) { + t.Fatal(i, j, brokenSerial) + } + + if g, e := b2n(k), down[j]; g != e { + t.Fatal(i, j, brokenSerial, g, e) + } + + if g, e := len(v), 8; g != e { + t.Fatal(i, g, e) + } + + if g, e := b2n(v), 10*down[j]; g != e { + t.Fatal(i, g, e) + } + + j++ + + } + + if g, e := j, len(down); g != e { + t.Fatal(i, j, g, e) + } + } + + } +} + +func TestBTreeSeekFirst(t *testing.T) { + db := NewBTree(nil) + en, err := db.SeekFirst() + if err == nil { + t.Fatal(err) + } + + if err := db.Set(n2b(100), n2b(1000)); err != nil { + t.Fatal(err) + } + + if en, err = db.SeekFirst(); err != nil { + t.Fatal(err) + } + + k, v, err := en.Next() + if err != nil { + t.Fatal(err) + } + + if g, e := b2n(k), 100; g != e { + t.Fatal(g, e) + } + + if g, e := b2n(v), 1000; g != e { + t.Fatal(g, e) + } + + if err := db.Set(n2b(110), n2b(1100)); err != nil { + t.Fatal(err) + } + + if en, err = db.SeekFirst(); err != nil { + t.Fatal(err) + } + + if k, v, err = en.Next(); err != nil { + t.Fatal(err) + } + + if g, e := b2n(k), 100; g != e { + t.Fatal(g, e) + } + + if g, e := b2n(v), 1000; g != e { + t.Fatal(g, e) + } + + if err := db.Set(n2b(90), n2b(900)); err != nil { + t.Fatal(err) + } + + if en, err = db.SeekFirst(); err != nil { + t.Fatal(err) + } + + if k, v, err = en.Next(); err != nil { + t.Fatal(err) + } + + if g, e := b2n(k), 90; g != e { + t.Fatal(g, e) + } + + if g, e := b2n(v), 900; g != e { + t.Fatal(g, e) + } + +} + +func TestBTreeSeekLast(t *testing.T) { + db := NewBTree(nil) + en, err := db.SeekLast() + if err == nil { + t.Fatal(err) + } + + if err := db.Set(n2b(100), n2b(1000)); err != nil { + t.Fatal(err) + } + + if en, err = db.SeekLast(); err != nil { + t.Fatal(err) + } + + k, v, err := en.Next() + if err != nil { + t.Fatal(err) + } + + if g, e := b2n(k), 100; g != e { + t.Fatal(g, e) + } + + if g, e := b2n(v), 1000; g != e { + t.Fatal(g, e) + } + + if err := db.Set(n2b(90), n2b(900)); err != nil { + t.Fatal(err) + } + + if en, err = db.SeekLast(); err != nil { + t.Fatal(err) + } + + if k, v, err = en.Next(); err != nil { + t.Fatal(err) + } + + if g, e := b2n(k), 100; g != e { + t.Fatal(g, e) + } + + if g, e := b2n(v), 1000; g != e { + t.Fatal(g, e) + } + + if err := db.Set(n2b(110), n2b(1100)); err != nil { + t.Fatal(err) + } + + if en, err = db.SeekLast(); err != nil { + t.Fatal(err) + } + + if k, v, err = en.Next(); err != nil { + t.Fatal(err) + } + + if g, e := b2n(k), 110; g != e { + t.Fatal(g, e) + } + + if g, e := b2n(v), 1100; g != e { + t.Fatal(g, e) + } + +} + +// https://code.google.com/p/camlistore/issues/detail?id=216 +func TestBug216(t *testing.T) { + const S = 2*kKV + 2 // 2*kKV+1 ok + const N = 300000 + rng, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + t.Fatal(err) + } + k := make([]byte, S/2) + v := make([]byte, S-S/2) + tr := NewBTree(nil) + for i := 0; i < N; i++ { + for i := range k { + k[i] = byte(rng.Next()) + } + for i := range v { + v[i] = byte(rng.Next()) + } + + if err := tr.Set(h2b(k, int64(i)), h2b(v, int64(i))); err != nil { + t.Fatal(i, err) + } + + if (i+1)%10000 == 0 { + //dbg("%v", i+1) + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/errors.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/errors.go new file mode 100644 index 00000000000..7dffe7f10cc --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/errors.go @@ -0,0 +1,170 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Some errors returned by this package. +// +// Note that this package can return more errors than declared here, for +// example io.EOF from Filer.ReadAt(). + +package lldb + +import ( + "fmt" +) + +// ErrDecodeScalars is possibly returned from DecodeScalars +type ErrDecodeScalars struct { + B []byte // Data being decoded + I int // offending offset +} + +// Error implements the built in error type. +func (e *ErrDecodeScalars) Error() string { + return fmt.Sprintf("DecodeScalars: corrupted data @ %d/%d", e.I, len(e.B)) +} + +// ErrINVAL reports invalid values passed as parameters, for example negative +// offsets where only non-negative ones are allowed or read from the DB. +type ErrINVAL struct { + Src string + Val interface{} +} + +// Error implements the built in error type. +func (e *ErrINVAL) Error() string { + return fmt.Sprintf("%s: %+v", e.Src, e.Val) +} + +// ErrPERM is for example reported when a Filer is closed while BeginUpdate(s) +// are not balanced with EndUpdate(s)/Rollback(s) or when EndUpdate or Rollback +// is invoked which is not paired with a BeginUpdate. +type ErrPERM struct { + Src string +} + +// Error implements the built in error type. +func (e *ErrPERM) Error() string { + return fmt.Sprintf("%s: Operation not permitted", string(e.Src)) +} + +// ErrTag represents an ErrILSEQ kind. +type ErrType int + +// ErrILSEQ types +const ( + ErrOther ErrType = iota + + ErrAdjacentFree // Adjacent free blocks (.Off and .Arg) + ErrDecompress // Used compressed block: corrupted compression + ErrExpFreeTag // Expected a free block tag, got .Arg + ErrExpUsedTag // Expected a used block tag, got .Arg + ErrFLT // Free block is invalid or referenced multiple times + ErrFLTLoad // FLT truncated to .Off, need size >= .Arg + ErrFLTSize // Free block size (.Arg) doesn't belong to its list min size: .Arg2 + ErrFileSize // File .Name size (.Arg) != 0 (mod 16) + ErrFreeChaining // Free block, .prev.next doesn't point back to this block + ErrFreeTailBlock // Last block is free + ErrHead // Head of a free block list has non zero Prev (.Arg) + ErrInvalidRelocTarget // Reloc doesn't target (.Arg) a short or long used block + ErrInvalidWAL // Corrupted write ahead log. .Name: file name, .More: more + ErrLongFreeBlkTooLong // Long free block spans beyond EOF, size .Arg + ErrLongFreeBlkTooShort // Long free block must have at least 2 atoms, got only .Arg + ErrLongFreeNextBeyondEOF // Long free block .Next (.Arg) spans beyond EOF + ErrLongFreePrevBeyondEOF // Long free block .Prev (.Arg) spans beyond EOF + ErrLongFreeTailTag // Expected a long free block tail tag, got .Arg + ErrLostFreeBlock // Free block is not in any FLT list + ErrNullReloc // Used reloc block with nil target + ErrRelocBeyondEOF // Used reloc points (.Arg) beyond EOF + ErrShortFreeTailTag // Expected a short free block tail tag, got .Arg + ErrSmall // Request for a free block (.Arg) returned a too small one (.Arg2) at .Off + ErrTailTag // Block at .Off has invalid tail CC (compression code) tag, got .Arg + ErrUnexpReloc // Unexpected reloc block referred to from reloc block .Arg + ErrVerifyPadding // Used block has nonzero padding + ErrVerifyTailSize // Long free block size .Arg but tail size .Arg2 + ErrVerifyUsedSpan // Used block size (.Arg) spans beyond EOF +) + +// ErrILSEQ reports a corrupted file format. Details in fields according to Type. +type ErrILSEQ struct { + Type ErrType + Off int64 + Arg int64 + Arg2 int64 + Arg3 int64 + Name string + More interface{} +} + +// Error implements the built in error type. +func (e *ErrILSEQ) Error() string { + switch e.Type { + case ErrAdjacentFree: + return fmt.Sprintf("Adjacent free blocks at offset %#x and %#x", e.Off, e.Arg) + case ErrDecompress: + return fmt.Sprintf("Compressed block at offset %#x: Corrupted compressed content", e.Off) + case ErrExpFreeTag: + return fmt.Sprintf("Block at offset %#x: Expected a free block tag, got %#2x", e.Off, e.Arg) + case ErrExpUsedTag: + return fmt.Sprintf("Block at ofset %#x: Expected a used block tag, got %#2x", e.Off, e.Arg) + case ErrFLT: + return fmt.Sprintf("Free block at offset %#x is invalid or referenced multiple times", e.Off) + case ErrFLTLoad: + return fmt.Sprintf("FLT truncated to size %d, expected at least %d", e.Off, e.Arg) + case ErrFLTSize: + return fmt.Sprintf("Free block at offset %#x has size (%#x) should be at least (%#x)", e.Off, e.Arg, e.Arg2) + case ErrFileSize: + return fmt.Sprintf("File %q size (%#x) != 0 (mod 16)", e.Name, e.Arg) + case ErrFreeChaining: + return fmt.Sprintf("Free block at offset %#x: .prev.next doesn point back here.", e.Off) + case ErrFreeTailBlock: + return fmt.Sprintf("Free block at offset %#x: Cannot be last file block", e.Off) + case ErrHead: + return fmt.Sprintf("Block at offset %#x: Head of free block list has non zero .prev %#x", e.Off, e.Arg) + case ErrInvalidRelocTarget: + return fmt.Sprintf("Used reloc block at offset %#x: Target (%#x) is not a short or long used block", e.Off, e.Arg) + case ErrInvalidWAL: + return fmt.Sprintf("Corrupted write ahead log file: %q %v", e.Name, e.More) + case ErrLongFreeBlkTooLong: + return fmt.Sprintf("Long free block at offset %#x: Size (%#x) beyond EOF", e.Off, e.Arg) + case ErrLongFreeBlkTooShort: + return fmt.Sprintf("Long free block at offset %#x: Size (%#x) too small", e.Off, e.Arg) + case ErrLongFreeNextBeyondEOF: + return fmt.Sprintf("Long free block at offset %#x: Next (%#x) points beyond EOF", e.Off, e.Arg) + case ErrLongFreePrevBeyondEOF: + return fmt.Sprintf("Long free block at offset %#x: Prev (%#x) points beyond EOF", e.Off, e.Arg) + case ErrLongFreeTailTag: + return fmt.Sprintf("Block at offset %#x: Expected long free tail tag, got %#2x", e.Off, e.Arg) + case ErrLostFreeBlock: + return fmt.Sprintf("Free block at offset %#x: not in any FLT list", e.Off) + case ErrNullReloc: + return fmt.Sprintf("Used reloc block at offset %#x: Nil target", e.Off) + case ErrRelocBeyondEOF: + return fmt.Sprintf("Used reloc block at offset %#x: Link (%#x) points beyond EOF", e.Off, e.Arg) + case ErrShortFreeTailTag: + return fmt.Sprintf("Block at offset %#x: Expected short free tail tag, got %#2x", e.Off, e.Arg) + case ErrSmall: + return fmt.Sprintf("Request for of free block of size %d returned a too small (%d) one at offset %#x", e.Arg, e.Arg2, e.Off) + case ErrTailTag: + return fmt.Sprintf("Block at offset %#x: Invalid tail CC tag, got %#2x", e.Off, e.Arg) + case ErrUnexpReloc: + return fmt.Sprintf("Block at offset %#x: Unexpected reloc block. Referred to from reloc block at offset %#x", e.Off, e.Arg) + case ErrVerifyPadding: + return fmt.Sprintf("Used block at offset %#x: Nonzero padding", e.Off) + case ErrVerifyTailSize: + return fmt.Sprintf("Long free block at offset %#x: Size %#x, but tail size %#x", e.Off, e.Arg, e.Arg2) + case ErrVerifyUsedSpan: + return fmt.Sprintf("Used block at offset %#x: Size %#x spans beyond EOF", e.Off, e.Arg) + } + + more := "" + if e.More != nil { + more = fmt.Sprintf(", %v", e.More) + } + off := "" + if e.Off != 0 { + off = fmt.Sprintf(", off: %#x", e.Off) + } + + return fmt.Sprintf("Error%s%s", off, more) +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/falloc.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/falloc.go new file mode 100644 index 00000000000..85638529f3a --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/falloc.go @@ -0,0 +1,1970 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The storage space management. + +package lldb + +import ( + "bytes" + "errors" + "fmt" + "io" + "sort" + "strings" + + "github.com/cznic/bufs" + "github.com/cznic/mathutil" + "github.com/cznic/zappy" +) + +const ( + maxBuf = maxRq + 20 // bufs,Buffers.Alloc +) + +// Options are passed to the NewAllocator to amend some configuration. The +// compatibility promise is the same as of struct types in the Go standard +// library - introducing changes can be made only by adding new exported +// fields, which is backward compatible as long as client code uses field names +// to assign values of imported struct types literals. +// +// NOTE: No options are currently defined. +type Options struct{} + +// AllocStats record statistics about a Filer. It can be optionally filled by +// Allocator.Verify, if successful. +type AllocStats struct { + Handles int64 // total valid handles in use + Compression int64 // number of compressed blocks + TotalAtoms int64 // total number of atoms == AllocAtoms + FreeAtoms + AllocBytes int64 // bytes allocated (after decompression, if/where used) + AllocAtoms int64 // atoms allocated/used, including relocation atoms + Relocations int64 // number of relocated used blocks + FreeAtoms int64 // atoms unused + AllocMap map[int64]int64 // allocated block size in atoms -> count of such blocks + FreeMap map[int64]int64 // free block size in atoms -> count of such blocks +} + +/* + +Allocator implements "raw" storage space management (allocation and +deallocation) for a low level of a DB engine. The storage is an abstraction +provided by a Filer. + +The terms MUST or MUST NOT, if/where used in the documentation of Allocator, +written in all caps as seen here, are a requirement for any possible +alternative implementations aiming for compatibility with this one. + +Filer file + +A Filer file, or simply 'file', is a linear, contiguous sequence of blocks. +Blocks may be either free (currently unused) or allocated (currently used). +Some blocks may eventually become virtual in a sense as they may not be +realized in the storage (sparse files). + +Free Lists Table + +File starts with a FLT. This table records heads of 14 doubly linked free +lists. The zero based index (I) vs minimal size of free blocks in that list, +except the last one which registers free blocks of size 4112+ atoms: + + MinSize == 2^I + + For example 0 -> 1, 1 -> 2, ... 12 -> 4096. + +Each entry in the FLT is 8 bytes in netwtork order, MSB MUST be zero, ie. the +slot value is effectively only 7 bytes. The value is the handle of the head of +the respective doubly linked free list. The FLT size is 14*8 == 112(0x70) +bytes. If the free blocks list for any particular size is empty, the respective +FLT slot is zero. Sizes of free blocks in one list MUST NOT overlap with sizes +of free lists in other list. For example, even though a free block of size 2 +technically is of minimal size >= 1, it MUST NOT be put to the list for slot 0 +(minimal size 1), but in slot 1( minimal size 2). + + slot 0: sizes [1, 2) + slot 1: sizes [2, 4) + slot 2: sizes [4, 8) + ... + slot 11: sizes [2048, 4096) + slot 12: sizes [4096, 4112) + slot 13: sizes [4112, inf) + +The last FLT slot collects all free blocks bigger than its minimal size. That +still respects the 'no overlap' invariant. + +File blocks + +A block is a linear, contiguous sequence of atoms. The first and last atoms of +a block provide information about, for example, whether the block is free or +used, what is the size of the block, etc. Details are discussed elsewhere. The +first block of a file starts immediately after FLT, ie. at file offset +112(0x70). + +Block atoms + +An atom is a fixed size piece of a block (and thus of a file too); it is 16 +bytes long. A consequence is that for a valid file: + + filesize == 0 (mod 16) + +The first atom of the first block is considered to be atom #1. + +Block handles + +A handle is an integer referring to a block. The reference is the number of the +atom the block starts with. Put in other way: + + handle == offset/16 - 6 + offset == 16 * (handle + 6) + +`offset` is the offset of the first byte of the block, measured in bytes +- as in fseek(3). Handle has type `int64`, but only the lower 7 bytes may be +nonzero while referring to a block, both in code as well as when persisted in +the the file's internal bookkeeping structures - see 'Block types' bellow. So a +handle is effectively only `uint56`. This also means that the maximum usable +size of a file is 2^56 atoms. That is 2^60 bytes == 1 exabyte (10^18 bytes). + +Nil handles + +A handle with numeric value of '0' refers to no block. + +Zero padding + +A padding is used to round-up a block size to be a whole number of atoms. Any +padding, if present, MUST be all zero bytes. Note that the size of padding is +in [0, 15]. + +Content wiping + +When a block is deallocated, its data content is not wiped as the added +overhead may be substantial while not necessarily needed. Client code should +however overwrite the content of any block having sensitive data with eg. zeros +(good compression) - before deallocating the block. + +Block tags + +Every block is tagged in its first byte (a head tag) and last byte (tail tag). +Block types are: + + 1. Short content used block (head tags 0x00-0xFB) + 2. Long content used block (head tag 0xFC) + 3. Relocated used block (head tag 0xFD) + 4. Short, single atom, free block (head tag 0xFE) + 5. Long free block (head tag 0xFF) + +Note: Relocated used block, 3. above (head tag 0xFD) MUST NOT refer to blocks +other then 1. or 2. above (head tags 0x00-0xFC). + +Content blocks + +Used blocks (head tags 0x00-0xFC) tail tag distinguish used/unused block and if +content is compressed or not. + +Content compression + +The tail flag of an used block is one of + + CC == 0 // Content is not compressed. + CC == 1 // Content is in zappy compression format. + +If compression of written content is enabled, there are two cases: If +compressed size < original size then the compressed content should be written +if it will save at least one atom of the block. If compressed size >= original +size then the compressed content should not be used. + +It's recommended to use compression. For example the BTrees implementation +assumes compression is used. Using compression may cause a slowdown in some +cases while it may as well cause a speedup. + +Short content block + +Short content block carries content of length between N == 0(0x00) and N == +251(0xFB) bytes. + + |<-first atom start ... last atom end->| + +---++-- ... --+-- ... --++------+ + | 0 || 1... | 0x*...0x*E || 0x*F | + +---++-- ... --+-- ... --++------+ + | N || content | padding || CC | + +---++-- ... --+-- ... --++------+ + + A == (N+1)/16 + 1 // The number of atoms in the block [1, 16] + padding == 15 - (N+1)%16 // Length of the zero padding + +Long content block + +Long content block carries content of length between N == 252(0xFC) and N == +65787(0x100FB) bytes. + + |<-first atom start ... last atom end->| + +------++------+-- ... --+-- ... --++------+ + | 0 || 1..2 | 3... | 0x*...0x*E || 0x*F | + +------++------+-- ... --+-- ... --++------+ + | 0xFC || M | content | padding || CC | + +------++------+-- ... --+-- ... --++------+ + + A == (N+3)/16 + 1 // The number of atoms in the block [16, 4112] + M == N % 0x10000 // Stored as 2 bytes in network byte order + padding == 15 - (N+3)%16 // Length of the zero padding + +Relocated used block + +Relocated block allows to permanently assign a handle to some content and +resize the content anytime afterwards without having to update all the possible +existing references; the handle can be constant while the content size may be +dynamic. When relocating a block, any space left by the original block content, +above this single atom block, MUST be reclaimed. + +Relocations MUST point only to a used short or long block == blocks with tags +0x00...0xFC. + + +------++------+---------++----+ + | 0 || 1..7 | 8...14 || 15 | + +------++------+---------++----+ + | 0xFD || H | padding || 0 | + +------++------+---------++----+ + +H is the handle of the relocated block in network byte order. + +Free blocks + +Free blocks are the result of space deallocation. Free blocks are organized in +one or more doubly linked lists, abstracted by the FLT interface. Free blocks +MUST be "registered" by putting them in such list. Allocator MUST reuse a big +enough free block, if such exists, before growing the file size. When a free +block is created by deallocation or reallocation it MUST be joined with any +adjacently existing free blocks before "registering". If the resulting free +block is now a last block of a file, the free block MUST be discarded and the +file size MUST be truncated accordingly instead. Put differently, there MUST +NOT ever be a free block at the file end. + +A single free atom + +Is an unused block of size 1 atom. + + +------++------+--------++------+ + | 0 || 1..7 | 8...14 || 15 | + +------++------+--------++------+ + | 0xFE || P | N || 0xFE | + +------++------+--------++------+ + +P and N, stored in network byte order, are the previous and next free block +handles in the doubly linked list to which this free block belongs. + +A long unused block + +Is an unused block of size > 1 atom. + + +------++------+-------+---------+- ... -+----------++------+ + | 0 || 1..7 | 8..14 | 15...21 | | Z-7..Z-1 || Z | + +------++------+-------+---------+- ... -+----------++------+ + | 0xFF || S | P | N | Leak | S || 0xFF | + +------++------+-------+---------+- ... -+----------++------+ + + Z == 16 * S - 1 + +S is the size of this unused block in atoms. P and N are the previous and next +free block handles in the doubly linked list to which this free block belongs. +Leak contains any data the block had before deallocating this block. See also +the subtitle 'Content wiping' above. S, P and N are stored in network byte +order. Large free blocks may trigger a consideration of file hole punching of +the Leak field - for some value of 'large'. + +Note: Allocator methods vs CRUD[1]: + + Alloc [C]reate + Get [R]ead + Realloc [U]pdate + Free [D]elete + +Note: No Allocator method returns io.EOF. + + [1]: http://en.wikipedia.org/wiki/Create,_read,_update_and_delete + +*/ +type Allocator struct { + f Filer + flt flt + Compress bool // enables content compression + cache cache + m map[int64]*node + lru lst + expHit int64 + expMiss int64 + cacheSz int + hit uint16 + miss uint16 +} + +// NewAllocator returns a new Allocator. To open an existing file, pass its +// Filer. To create a "new" file, pass a Filer which file is of zero size. +func NewAllocator(f Filer, opts *Options) (a *Allocator, err error) { + if opts == nil { // Enforce *Options is always passed + return nil, errors.New("NewAllocator: nil opts passed") + } + + a = &Allocator{ + f: f, + cacheSz: 10, + } + + a.cinit() + switch x := f.(type) { + case *RollbackFiler: + x.afterRollback = func() error { + a.cinit() + return a.flt.load(a.f, 0) + } + case *ACIDFiler0: + x.RollbackFiler.afterRollback = func() error { + a.cinit() + return a.flt.load(a.f, 0) + } + } + + sz, err := f.Size() + if err != nil { + return + } + + a.flt.init() + if sz == 0 { + var b [fltSz]byte + if err = a.f.BeginUpdate(); err != nil { + return + } + + if _, err = f.WriteAt(b[:], 0); err != nil { + a.f.Rollback() + return + } + + return a, a.f.EndUpdate() + } + + return a, a.flt.load(f, 0) +} + +// CacheStats reports cache statistics. +// +//TODO return a struct perhaps. +func (a *Allocator) CacheStats() (buffersUsed, buffersTotal int, bytesUsed, bytesTotal, hits, misses int64) { + buffersUsed = len(a.m) + buffersTotal = buffersUsed + len(a.cache) + bytesUsed = a.lru.size() + bytesTotal = bytesUsed + a.cache.size() + hits = a.expHit + misses = a.expMiss + return +} + +func (a *Allocator) cinit() { + for h, n := range a.m { + a.cache.put(a.lru.remove(n)) + delete(a.m, h) + } + if a.m == nil { + a.m = map[int64]*node{} + } +} + +func (a *Allocator) cadd(b []byte, h int64) { + if len(a.m) < a.cacheSz { + n := a.cache.get(len(b)) + n.h = h + copy(n.b, b) + a.m[h] = a.lru.pushFront(n) + return + } + + // cache full + delete(a.m, a.cache.put(a.lru.removeBack()).h) + n := a.cache.get(len(b)) + n.h = h + copy(n.b, b) + a.m[h] = a.lru.pushFront(n) + return +} + +func (a *Allocator) cfree(h int64) { + n, ok := a.m[h] + if !ok { // must have been evicted + return + } + + a.cache.put(a.lru.remove(n)) + delete(a.m, h) +} + +// Alloc allocates storage space for b and returns the handle of the new block +// with content set to b or an error, if any. The returned handle is valid only +// while the block is used - until the block is deallocated. No two valid +// handles share the same value within the same Filer, but any value of a +// handle not referring to any used block may become valid any time as a result +// of Alloc. +// +// Invoking Alloc on an empty Allocator is guaranteed to return handle with +// value 1. The intended use of content of handle 1 is a root "directory" of +// other data held by an Allocator. +// +// Passing handles not obtained initially from Alloc or not anymore valid to +// any other Allocator methods can result in an irreparably corrupted database. +func (a *Allocator) Alloc(b []byte) (handle int64, err error) { + buf := bufs.GCache.Get(zappy.MaxEncodedLen(len(b))) + defer bufs.GCache.Put(buf) + buf, _, cc, err := a.makeUsedBlock(buf, b) + if err != nil { + return + } + + if handle, err = a.alloc(buf, cc); err == nil { + a.cadd(b, handle) + } + return +} + +func (a *Allocator) alloc(b []byte, cc byte) (h int64, err error) { + rqAtoms := n2atoms(len(b)) + if h = a.flt.find(rqAtoms); h == 0 { // must grow + var sz int64 + if sz, err = a.f.Size(); err != nil { + return + } + + h = off2h(sz) + err = a.writeUsedBlock(h, cc, b) + return + } + + // Handle is the first item of a free blocks list. + tag, s, prev, next, err := a.nfo(h) + if err != nil { + return + } + + if tag != tagFreeShort && tag != tagFreeLong { + err = &ErrILSEQ{Type: ErrExpFreeTag, Off: h2off(h), Arg: int64(tag)} + return + } + + if prev != 0 { + err = &ErrILSEQ{Type: ErrHead, Off: h2off(h), Arg: prev} + return + } + + if s < int64(rqAtoms) { + err = &ErrILSEQ{Type: ErrSmall, Arg: int64(rqAtoms), Arg2: s, Off: h2off(h)} + return + } + + if err = a.unlink(h, s, prev, next); err != nil { + return + } + + if s > int64(rqAtoms) { + freeH := h + int64(rqAtoms) + freeAtoms := s - int64(rqAtoms) + if err = a.link(freeH, freeAtoms); err != nil { + return + } + } + return h, a.writeUsedBlock(h, cc, b) +} + +// Free deallocates the block referred to by handle or returns an error, if +// any. +// +// After Free succeeds, handle is invalid and must not be used. +// +// Handle must have been obtained initially from Alloc and must be still valid, +// otherwise a database may get irreparably corrupted. +func (a *Allocator) Free(handle int64) (err error) { + if handle <= 0 || handle > maxHandle { + return &ErrINVAL{"Allocator.Free: handle out of limits", handle} + } + + a.cfree(handle) + return a.free(handle, 0, true) +} + +func (a *Allocator) free(h, from int64, acceptRelocs bool) (err error) { + tag, atoms, _, n, err := a.nfo(h) + if err != nil { + return + } + + switch tag { + default: + // nop + case tagUsedLong: + // nop + case tagUsedRelocated: + if !acceptRelocs { + return &ErrILSEQ{Type: ErrUnexpReloc, Off: h2off(h), Arg: h2off(from)} + } + + if err = a.free(n, h, false); err != nil { + return + } + case tagFreeShort, tagFreeLong: + return &ErrINVAL{"Allocator.Free: attempt to free a free block at off", h2off(h)} + } + + return a.free2(h, atoms) +} + +func (a *Allocator) free2(h, atoms int64) (err error) { + sz, err := a.f.Size() + if err != nil { + return + } + + ltag, latoms, lp, ln, err := a.leftNfo(h) + if err != nil { + return + } + + if ltag != tagFreeShort && ltag != tagFreeLong { + latoms = 0 + } + + var rtag byte + var ratoms, rp, rn int64 + + isTail := h2off(h)+atoms*16 == sz + if !isTail { + if rtag, ratoms, rp, rn, err = a.nfo(h + atoms); err != nil { + return + } + } + + if rtag != tagFreeShort && rtag != tagFreeLong { + ratoms = 0 + } + + switch { + case latoms == 0 && ratoms == 0: + // -> isolated <- + if isTail { // cut tail + return a.f.Truncate(h2off(h)) + } + + return a.link(h, atoms) + case latoms == 0 && ratoms != 0: + // right join -> + if err = a.unlink(h+atoms, ratoms, rp, rn); err != nil { + return + } + + return a.link(h, atoms+ratoms) + case latoms != 0 && ratoms == 0: + // <- left join + if err = a.unlink(h-latoms, latoms, lp, ln); err != nil { + return + } + + if isTail { + return a.f.Truncate(h2off(h - latoms)) + } + + return a.link(h-latoms, latoms+atoms) + } + + // case latoms != 0 && ratoms != 0: + // <- middle join -> + lh, rh := h-latoms, h+atoms + if err = a.unlink(lh, latoms, lp, ln); err != nil { + return + } + + // Prev unlink may have invalidated rp or rn + if _, _, rp, rn, err = a.nfo(rh); err != nil { + return + } + + if err = a.unlink(rh, ratoms, rp, rn); err != nil { + return + } + + return a.link(h-latoms, latoms+atoms+ratoms) +} + +// Add a free block h to the appropriate free list +func (a *Allocator) link(h, atoms int64) (err error) { + if err = a.makeFree(h, atoms, 0, a.flt.head(atoms)); err != nil { + return + } + + return a.flt.setHead(h, atoms, a.f) +} + +// Remove free block h from the free list +func (a *Allocator) unlink(h, atoms, p, n int64) (err error) { + switch { + case p == 0 && n == 0: + // single item list, must be head + return a.flt.setHead(0, atoms, a.f) + case p == 0 && n != 0: + // head of list (has next item[s]) + if err = a.prev(n, 0); err != nil { + return + } + + // new head + return a.flt.setHead(n, atoms, a.f) + case p != 0 && n == 0: + // last item in list + return a.next(p, 0) + } + // case p != 0 && n != 0: + // intermediate item in a list + if err = a.next(p, n); err != nil { + return + } + + return a.prev(n, p) +} + +//TODO remove ? +// Return len(slice) == n, reuse src if possible. +func need(n int, src []byte) []byte { + if cap(src) < n { + bufs.GCache.Put(src) + return bufs.GCache.Get(n) + } + + return src[:n] +} + +// Get returns the data content of a block referred to by handle or an error if +// any. The returned slice may be a sub-slice of buf if buf was large enough +// to hold the entire content. Otherwise, a newly allocated slice will be +// returned. It is valid to pass a nil buf. +// +// If the content was stored using compression then it is transparently +// returned decompressed. +// +// Handle must have been obtained initially from Alloc and must be still valid, +// otherwise invalid data may be returned without detecting the error. +func (a *Allocator) Get(buf []byte, handle int64) (b []byte, err error) { + buf = buf[:cap(buf)] + if n, ok := a.m[handle]; ok { + a.lru.moveToFront(n) + b = need(len(n.b), buf) + copy(b, n.b) + a.expHit++ + a.hit++ + return + } + + a.expMiss++ + a.miss++ + if a.miss > 10 && len(a.m) < 500 { + if 100*a.hit/a.miss < 95 { + a.cacheSz++ + } + a.hit, a.miss = 0, 0 + } + defer func(h int64) { + if err == nil { + a.cadd(b, h) + } + }(handle) + + first := bufs.GCache.Get(16) + defer bufs.GCache.Put(first) + relocated := false + relocSrc := handle +reloc: + if handle <= 0 || handle > maxHandle { + return nil, &ErrINVAL{"Allocator.Get: handle out of limits", handle} + } + + off := h2off(handle) + if err = a.read(first, off); err != nil { + return + } + + switch tag := first[0]; tag { + default: + dlen := int(tag) + atoms := n2atoms(dlen) + switch atoms { + case 1: + switch tag := first[15]; tag { + default: + return nil, &ErrILSEQ{Type: ErrTailTag, Off: off, Arg: int64(tag)} + case tagNotCompressed: + b = need(dlen, buf) + copy(b, first[1:]) + return + case tagCompressed: + return zappy.Decode(buf, first[1:dlen+1]) + } + default: + cc := bufs.GCache.Get(1) + defer bufs.GCache.Put(cc) + dlen := int(tag) + atoms := n2atoms(dlen) + tailOff := off + 16*int64(atoms) - 1 + if err = a.read(cc, tailOff); err != nil { + return + } + + switch tag := cc[0]; tag { + default: + return nil, &ErrILSEQ{Type: ErrTailTag, Off: off, Arg: int64(tag)} + case tagNotCompressed: + b = need(dlen, buf) + off += 1 + if err = a.read(b, off); err != nil { + b = buf[:0] + } + return + case tagCompressed: + zbuf := bufs.GCache.Get(dlen) + defer bufs.GCache.Put(zbuf) + off += 1 + if err = a.read(zbuf, off); err != nil { + return buf[:0], err + } + + return zappy.Decode(buf, zbuf) + } + } + case 0: + return buf[:0], nil + case tagUsedLong: + cc := bufs.GCache.Get(1) + defer bufs.GCache.Put(cc) + dlen := m2n(int(first[1])<<8 | int(first[2])) + atoms := n2atoms(dlen) + tailOff := off + 16*int64(atoms) - 1 + if err = a.read(cc, tailOff); err != nil { + return + } + + switch tag := cc[0]; tag { + default: + return nil, &ErrILSEQ{Type: ErrTailTag, Off: off, Arg: int64(tag)} + case tagNotCompressed: + b = need(dlen, buf) + off += 3 + if err = a.read(b, off); err != nil { + b = buf[:0] + } + return + case tagCompressed: + zbuf := bufs.GCache.Get(dlen) + defer bufs.GCache.Put(zbuf) + off += 3 + if err = a.read(zbuf, off); err != nil { + return buf[:0], err + } + + return zappy.Decode(buf, zbuf) + } + case tagFreeShort, tagFreeLong: + return nil, &ErrILSEQ{Type: ErrExpUsedTag, Off: off, Arg: int64(tag)} + case tagUsedRelocated: + if relocated { + return nil, &ErrILSEQ{Type: ErrUnexpReloc, Off: off, Arg: relocSrc} + } + + handle = b2h(first[1:]) + relocated = true + goto reloc + } +} + +var reallocTestHook bool + +// Realloc sets the content of a block referred to by handle or returns an +// error, if any. +// +// Handle must have been obtained initially from Alloc and must be still valid, +// otherwise a database may get irreparably corrupted. +func (a *Allocator) Realloc(handle int64, b []byte) (err error) { + if handle <= 0 || handle > maxHandle { + return &ErrINVAL{"Realloc: handle out of limits", handle} + } + + a.cfree(handle) + if err = a.realloc(handle, b); err != nil { + return + } + + if reallocTestHook { + if err = cacheAudit(a.m, &a.lru); err != nil { + return + } + } + + a.cadd(b, handle) + return +} + +func (a *Allocator) realloc(handle int64, b []byte) (err error) { + var dlen, needAtoms0 int + + b8 := bufs.GCache.Get(8) + defer bufs.GCache.Put(b8) + dst := bufs.GCache.Get(zappy.MaxEncodedLen(len(b))) + defer bufs.GCache.Put(dst) + b, needAtoms0, cc, err := a.makeUsedBlock(dst, b) + if err != nil { + return + } + + needAtoms := int64(needAtoms0) + off := h2off(handle) + if err = a.read(b8[:], off); err != nil { + return + } + + switch tag := b8[0]; tag { + default: + dlen = int(b8[0]) + case tagUsedLong: + dlen = m2n(int(b8[1])<<8 | int(b8[2])) + case tagUsedRelocated: + if err = a.free(b2h(b8[1:]), handle, false); err != nil { + return err + } + + dlen = 0 + case tagFreeShort, tagFreeLong: + return &ErrINVAL{"Allocator.Realloc: invalid handle", handle} + } + + atoms := int64(n2atoms(dlen)) +retry: + switch { + case needAtoms < atoms: + // in place shrink + if err = a.writeUsedBlock(handle, cc, b); err != nil { + return + } + + fh, fa := handle+needAtoms, atoms-needAtoms + sz, err := a.f.Size() + if err != nil { + return err + } + + if h2off(fh)+16*fa == sz { + return a.f.Truncate(h2off(fh)) + } + + return a.free2(fh, fa) + case needAtoms == atoms: + // in place replace + return a.writeUsedBlock(handle, cc, b) + } + + // case needAtoms > atoms: + // in place extend or relocate + var sz int64 + if sz, err = a.f.Size(); err != nil { + return + } + + off = h2off(handle) + switch { + case off+atoms*16 == sz: + // relocating tail block - shortcut + return a.writeUsedBlock(handle, cc, b) + default: + if off+atoms*16 < sz { + // handle is not a tail block, check right neighbour + rh := handle + atoms + rtag, ratoms, p, n, e := a.nfo(rh) + if e != nil { + return e + } + + if rtag == tagFreeShort || rtag == tagFreeLong { + // Right neighbour is a free block + if needAtoms <= atoms+ratoms { + // can expand in place + if err = a.unlink(rh, ratoms, p, n); err != nil { + return + } + + atoms += ratoms + goto retry + + } + } + } + } + + if atoms > 1 { + if err = a.realloc(handle, nil); err != nil { + return + } + } + + var newH int64 + if newH, err = a.alloc(b, cc); err != nil { + return err + } + + rb := bufs.GCache.Cget(16) + defer bufs.GCache.Put(rb) + rb[0] = tagUsedRelocated + h2b(rb[1:], newH) + if err = a.writeAt(rb[:], h2off(handle)); err != nil { + return + } + + return a.writeUsedBlock(newH, cc, b) +} + +func (a *Allocator) writeAt(b []byte, off int64) (err error) { + var n int + if n, err = a.f.WriteAt(b, off); err != nil { + return + } + + if n != len(b) { + err = io.ErrShortWrite + } + return +} + +func (a *Allocator) write(off int64, b ...[]byte) (err error) { + rq := 0 + for _, part := range b { + rq += len(part) + } + buf := bufs.GCache.Get(rq) + defer bufs.GCache.Put(buf) + buf = buf[:0] + for _, part := range b { + buf = append(buf, part...) + } + return a.writeAt(buf, off) +} + +func (a *Allocator) read(b []byte, off int64) (err error) { + var rn int + if rn, err = a.f.ReadAt(b, off); rn != len(b) { + return &ErrILSEQ{Type: ErrOther, Off: off, More: err} + } + + return nil +} + +// nfo returns h's tag. If it's a free block then return also (s)ize (in +// atoms), (p)rev and (n)ext. If it's a used block then only (s)ize is returned +// (again in atoms). If it's a used relocate block then (n)ext is set to the +// relocation target handle. +func (a *Allocator) nfo(h int64) (tag byte, s, p, n int64, err error) { + off := h2off(h) + rq := int64(22) + sz, err := a.f.Size() + if err != nil { + return + } + + if off+rq >= sz { + if rq = sz - off; rq < 15 { + err = io.ErrUnexpectedEOF + return + } + } + + buf := bufs.GCache.Get(22) + defer bufs.GCache.Put(buf) + if err = a.read(buf[:rq], off); err != nil { + return + } + + switch tag = buf[0]; tag { + default: + s = int64(n2atoms(int(tag))) + case tagUsedLong: + s = int64(n2atoms(m2n(int(buf[1])<<8 | int(buf[2])))) + case tagFreeLong: + if rq < 22 { + err = io.ErrUnexpectedEOF + return + } + + s, p, n = b2h(buf[1:]), b2h(buf[8:]), b2h(buf[15:]) + case tagUsedRelocated: + s, n = 1, b2h(buf[1:]) + case tagFreeShort: + s, p, n = 1, b2h(buf[1:]), b2h(buf[8:]) + } + return +} + +// leftNfo returns nfo for h's left neighbor if h > 1 and the left neighbor is +// a free block. Otherwise all zero values are returned instead. +func (a *Allocator) leftNfo(h int64) (tag byte, s, p, n int64, err error) { + if !(h > 1) { + return + } + + buf := bufs.GCache.Get(8) + defer bufs.GCache.Put(buf) + off := h2off(h) + if err = a.read(buf[:], off-8); err != nil { + return + } + + switch tag := buf[7]; tag { + case tagFreeShort: + return a.nfo(h - 1) + case tagFreeLong: + return a.nfo(h - b2h(buf[:])) + } + return +} + +// Set h.prev = p +func (a *Allocator) prev(h, p int64) (err error) { + b := bufs.GCache.Get(7) + defer bufs.GCache.Put(b) + off := h2off(h) + if err = a.read(b[:1], off); err != nil { + return + } + + switch tag := b[0]; tag { + default: + return &ErrILSEQ{Type: ErrExpFreeTag, Off: off, Arg: int64(tag)} + case tagFreeShort: + off += 1 + case tagFreeLong: + off += 8 + } + return a.writeAt(h2b(b[:7], p), off) +} + +// Set h.next = n +func (a *Allocator) next(h, n int64) (err error) { + b := bufs.GCache.Get(7) + defer bufs.GCache.Put(b) + off := h2off(h) + if err = a.read(b[:1], off); err != nil { + return + } + + switch tag := b[0]; tag { + default: + return &ErrILSEQ{Type: ErrExpFreeTag, Off: off, Arg: int64(tag)} + case tagFreeShort: + off += 8 + case tagFreeLong: + off += 15 + } + return a.writeAt(h2b(b[:7], n), off) +} + +// Make the filer image @h a free block. +func (a *Allocator) makeFree(h, atoms, prev, next int64) (err error) { + buf := bufs.GCache.Get(22) + defer bufs.GCache.Put(buf) + switch { + case atoms == 1: + buf[0], buf[15] = tagFreeShort, tagFreeShort + h2b(buf[1:], prev) + h2b(buf[8:], next) + if err = a.write(h2off(h), buf[:16]); err != nil { + return + } + default: + + buf[0] = tagFreeLong + h2b(buf[1:], atoms) + h2b(buf[8:], prev) + h2b(buf[15:], next) + if err = a.write(h2off(h), buf[:22]); err != nil { + return + } + + h2b(buf[:], atoms) + buf[7] = tagFreeLong + if err = a.write(h2off(h+atoms)-8, buf[:8]); err != nil { + return + } + } + if prev != 0 { + if err = a.next(prev, h); err != nil { + return + } + } + + if next != 0 { + err = a.prev(next, h) + } + return +} + +func (a *Allocator) makeUsedBlock(dst []byte, b []byte) (w []byte, rqAtoms int, cc byte, err error) { + cc = tagNotCompressed + w = b + + var n int + if n = len(b); n > maxRq { + return nil, 0, 0, &ErrINVAL{"Allocator.makeUsedBlock: content size out of limits", n} + } + + rqAtoms = n2atoms(n) + if a.Compress && n > 14 { // attempt compression + if dst, err = zappy.Encode(dst, b); err != nil { + return + } + + n2 := len(dst) + if rqAtoms2 := n2atoms(n2); rqAtoms2 < rqAtoms { // compression saved at least a single atom + w, n, rqAtoms, cc = dst, n2, rqAtoms2, tagCompressed + } + } + return +} + +func (a *Allocator) writeUsedBlock(h int64, cc byte, b []byte) (err error) { + n := len(b) + rq := n2atoms(n) << 4 + buf := bufs.GCache.Get(rq) + defer bufs.GCache.Put(buf) + switch n <= maxShort { + case true: + buf[0] = byte(n) + copy(buf[1:], b) + case false: + m := n2m(n) + buf[0], buf[1], buf[2] = tagUsedLong, byte(m>>8), byte(m) + copy(buf[3:], b) + } + if p := n2padding(n); p != 0 { + copy(buf[rq-1-p:], zeros[:]) + } + buf[rq-1] = cc + return a.writeAt(buf, h2off(h)) +} + +func (a *Allocator) verifyUnused(h, totalAtoms int64, tag byte, log func(error) bool, fast bool) (atoms, prev, next int64, err error) { + switch tag { + default: + panic("internal error") + case tagFreeShort: + var b [16]byte + off := h2off(h) + if err = a.read(b[:], off); err != nil { + return + } + + if b[15] != tagFreeShort { + err = &ErrILSEQ{Type: ErrShortFreeTailTag, Off: off, Arg: int64(b[15])} + log(err) + return + } + + atoms, prev, next = 1, b2h(b[1:]), b2h(b[8:]) + case tagFreeLong: + var b [22]byte + off := h2off(h) + if err = a.read(b[:], off); err != nil { + return + } + + atoms, prev, next = b2h(b[1:]), b2h(b[8:]), b2h(b[15:]) + if fast { + return + } + + if atoms < 2 { + err = &ErrILSEQ{Type: ErrLongFreeBlkTooShort, Off: off, Arg: int64(atoms)} + break + } + + if h+atoms-1 > totalAtoms { + err = &ErrILSEQ{Type: ErrLongFreeBlkTooLong, Off: off, Arg: atoms} + break + } + + if prev > totalAtoms { + err = &ErrILSEQ{Type: ErrLongFreePrevBeyondEOF, Off: off, Arg: next} + break + } + + if next > totalAtoms { + err = &ErrILSEQ{Type: ErrLongFreeNextBeyondEOF, Off: off, Arg: next} + break + } + + toff := h2off(h+atoms) - 8 + if err = a.read(b[:8], toff); err != nil { + return + } + + if b[7] != tag { + err = &ErrILSEQ{Type: ErrLongFreeTailTag, Off: off, Arg: int64(b[7])} + break + } + + if s2 := b2h(b[:]); s2 != atoms { + err = &ErrILSEQ{Type: ErrVerifyTailSize, Off: off, Arg: atoms, Arg2: s2} + break + } + + } + if err != nil { + log(err) + } + return +} + +func (a *Allocator) verifyUsed(h, totalAtoms int64, tag byte, buf, ubuf []byte, log func(error) bool, fast bool) (compressed bool, dlen int, atoms, link int64, err error) { + var ( + padding int + doff int64 + padZeros [15]byte + tailBuf [16]byte + ) + + switch tag { + default: // Short used + dlen = int(tag) + atoms = int64((dlen+1)/16) + 1 + padding = 15 - (dlen+1)%16 + doff = h2off(h) + 1 + case tagUsedLong: + off := h2off(h) + 1 + var b2 [2]byte + if err = a.read(b2[:], off); err != nil { + return + } + + dlen = m2n(int(b2[0])<<8 | int(b2[1])) + atoms = int64((dlen+3)/16) + 1 + padding = 15 - (dlen+3)%16 + doff = h2off(h) + 3 + case tagUsedRelocated: + dlen = 7 + atoms = 1 + padding = 7 + doff = h2off(h) + 1 + case tagFreeShort, tagFreeLong: + panic("internal error") + } + + if fast { + if tag == tagUsedRelocated { + dlen = 0 + if err = a.read(buf[:7], doff); err != nil { + return + } + + link = b2h(buf) + } + + return false, dlen, atoms, link, nil + } + + if ok := h+atoms-1 <= totalAtoms; !ok { // invalid last block + err = &ErrILSEQ{Type: ErrVerifyUsedSpan, Off: h2off(h), Arg: atoms} + log(err) + return + } + + tailsz := 1 + padding + off := h2off(h) + 16*atoms - int64(tailsz) + if err = a.read(tailBuf[:tailsz], off); err != nil { + return false, 0, 0, 0, err + } + + if ok := bytes.Equal(padZeros[:padding], tailBuf[:padding]); !ok { + err = &ErrILSEQ{Type: ErrVerifyPadding, Off: h2off(h)} + log(err) + return + } + + var cc byte + switch cc = tailBuf[padding]; cc { + default: + err = &ErrILSEQ{Type: ErrTailTag, Off: h2off(h)} + log(err) + return + case tagCompressed: + compressed = true + if tag == tagUsedRelocated { + err = &ErrILSEQ{Type: ErrTailTag, Off: h2off(h)} + log(err) + return + } + + fallthrough + case tagNotCompressed: + if err = a.read(buf[:dlen], doff); err != nil { + return false, 0, 0, 0, err + } + } + + if cc == tagCompressed { + if ubuf, err = zappy.Decode(ubuf, buf[:dlen]); err != nil || len(ubuf) > maxRq { + err = &ErrILSEQ{Type: ErrDecompress, Off: h2off(h)} + log(err) + return + } + + dlen = len(ubuf) + } + + if tag == tagUsedRelocated { + link = b2h(buf) + if link == 0 { + err = &ErrILSEQ{Type: ErrNullReloc, Off: h2off(h)} + log(err) + return + } + + if link > totalAtoms { // invalid last block + err = &ErrILSEQ{Type: ErrRelocBeyondEOF, Off: h2off(h), Arg: link} + log(err) + return + } + } + + return +} + +var nolog = func(error) bool { return false } + +// Verify attempts to find any structural errors in a Filer wrt the +// organization of it as defined by Allocator. 'bitmap' is a scratch pad for +// necessary bookkeeping and will grow to at most to Allocator's +// Filer.Size()/128 (0,78%). Any problems found are reported to 'log' except +// non verify related errors like disk read fails etc. If 'log' returns false +// or the error doesn't allow to (reliably) continue, the verification process +// is stopped and an error is returned from the Verify function. Passing a nil +// log works like providing a log function always returning false. Any +// non-structural errors, like for instance Filer read errors, are NOT reported +// to 'log', but returned as the Verify's return value, because Verify cannot +// proceed in such cases. Verify returns nil only if it fully completed +// verifying Allocator's Filer without detecting any error. +// +// It is recommended to limit the number reported problems by returning false +// from 'log' after reaching some limit. Huge and corrupted DB can produce an +// overwhelming error report dataset. +// +// The verifying process will scan the whole DB at least 3 times (a trade +// between processing space and time consumed). It doesn't read the content of +// free blocks above the head/tail info bytes. If the 3rd phase detects lost +// free space, then a 4th scan (a faster one) is performed to precisely report +// all of them. +// +// If the DB/Filer to be verified is reasonably small, respective if its +// size/128 can comfortably fit within process's free memory, then it is +// recommended to consider using a MemFiler for the bit map. +// +// Statistics are returned via 'stats' if non nil. The statistics are valid +// only if Verify succeeded, ie. it didn't reported anything to log and it +// returned a nil error. +func (a *Allocator) Verify(bitmap Filer, log func(error) bool, stats *AllocStats) (err error) { + if log == nil { + log = nolog + } + + n, err := bitmap.Size() + if err != nil { + return + } + + if n != 0 { + return &ErrINVAL{"Allocator.Verify: bit map initial size non zero (%d)", n} + } + + var bits int64 + bitMask := [8]byte{1, 2, 4, 8, 16, 32, 64, 128} + byteBuf := []byte{0} + + //DONE + // +performance, this implementation is hopefully correct but _very_ + // naive, probably good as a prototype only. Use maybe a MemFiler + // "cache" etc. + // ---- + // Turns out the OS caching is as effective as it can probably get. + bit := func(on bool, h int64) (wasOn bool, err error) { + m := bitMask[h&7] + off := h >> 3 + var v byte + sz, err := bitmap.Size() + if err != nil { + return + } + + if off < sz { + if n, err := bitmap.ReadAt(byteBuf, off); n != 1 { + return false, &ErrILSEQ{Type: ErrOther, Off: off, More: fmt.Errorf("Allocator.Verify - reading bitmap: %s", err)} + } + + v = byteBuf[0] + } + switch wasOn = v&m != 0; on { + case true: + if !wasOn { + v |= m + bits++ + } + case false: + if wasOn { + v ^= m + bits-- + } + } + byteBuf[0] = v + if n, err := bitmap.WriteAt(byteBuf, off); n != 1 || err != nil { + return false, &ErrILSEQ{Type: ErrOther, Off: off, More: fmt.Errorf("Allocator.Verify - writing bitmap: %s", err)} + } + + return + } + + // Phase 1 - sequentially scan a.f to reliably determine block + // boundaries. Set a bit for every block start. + var ( + buf, ubuf [maxRq]byte + prevH, h, atoms int64 + wasOn bool + tag byte + st = AllocStats{ + AllocMap: map[int64]int64{}, + FreeMap: map[int64]int64{}, + } + dlen int + ) + + fsz, err := a.f.Size() + if err != nil { + return + } + + ok := fsz%16 == 0 + totalAtoms := (fsz - fltSz) / atomLen + if !ok { + err = &ErrILSEQ{Type: ErrFileSize, Name: a.f.Name(), Arg: fsz} + log(err) + return + } + + st.TotalAtoms = totalAtoms + prevTag := -1 + lastH := int64(-1) + + for h = 1; h <= totalAtoms; h += atoms { + prevH = h // For checking last block == used + + off := h2off(h) + if err = a.read(buf[:1], off); err != nil { + return + } + + switch tag = buf[0]; tag { + default: // Short used + fallthrough + case tagUsedLong, tagUsedRelocated: + var compressed bool + if compressed, dlen, atoms, _, err = a.verifyUsed(h, totalAtoms, tag, buf[:], ubuf[:], log, false); err != nil { + return + } + + if compressed { + st.Compression++ + } + st.AllocAtoms += atoms + switch { + case tag == tagUsedRelocated: + st.AllocMap[1]++ + st.Relocations++ + default: + st.AllocMap[atoms]++ + st.AllocBytes += int64(dlen) + st.Handles++ + } + case tagFreeShort, tagFreeLong: + if prevTag == tagFreeShort || prevTag == tagFreeLong { + err = &ErrILSEQ{Type: ErrAdjacentFree, Off: h2off(lastH), Arg: off} + log(err) + return + } + + if atoms, _, _, err = a.verifyUnused(h, totalAtoms, tag, log, false); err != nil { + return + } + + st.FreeMap[atoms]++ + st.FreeAtoms += atoms + } + + if wasOn, err = bit(true, h); err != nil { + return + } + + if wasOn { + panic("internal error") + } + + prevTag = int(tag) + lastH = h + } + + if totalAtoms != 0 && (tag == tagFreeShort || tag == tagFreeLong) { + err = &ErrILSEQ{Type: ErrFreeTailBlock, Off: h2off(prevH)} + log(err) + return + } + + // Phase 2 - check used blocks, turn off the map bit for every used + // block. + for h = 1; h <= totalAtoms; h += atoms { + off := h2off(h) + if err = a.read(buf[:1], off); err != nil { + return + } + + var link int64 + switch tag = buf[0]; tag { + default: // Short used + fallthrough + case tagUsedLong, tagUsedRelocated: + if _, _, atoms, link, err = a.verifyUsed(h, totalAtoms, tag, buf[:], ubuf[:], log, true); err != nil { + return + } + case tagFreeShort, tagFreeLong: + if atoms, _, _, err = a.verifyUnused(h, totalAtoms, tag, log, true); err != nil { + return + } + } + + turnoff := true + switch tag { + case tagUsedRelocated: + if err = a.read(buf[:1], h2off(link)); err != nil { + return + } + + switch linkedTag := buf[0]; linkedTag { + case tagFreeShort, tagFreeLong, tagUsedRelocated: + err = &ErrILSEQ{Type: ErrInvalidRelocTarget, Off: off, Arg: link} + log(err) + return + } + + case tagFreeShort, tagFreeLong: + turnoff = false + } + + if !turnoff { + continue + } + + if wasOn, err = bit(false, h); err != nil { + return + } + + if !wasOn { + panic("internal error") + } + + } + + // Phase 3 - using the flt check heads link to proper free blocks. For + // every free block, walk the list, verify the {next, prev} links and + // turn the respective map bit off. After processing all free lists, + // the map bits count should be zero. Otherwise there are "lost" free + // blocks. + + var prev, next, fprev, fnext int64 + rep := a.flt + + for _, list := range rep { + prev, next = 0, list.head + for ; next != 0; prev, next = next, fnext { + if wasOn, err = bit(false, next); err != nil { + return + } + + if !wasOn { + err = &ErrILSEQ{Type: ErrFLT, Off: h2off(next), Arg: h} + log(err) + return + } + + off := h2off(next) + if err = a.read(buf[:1], off); err != nil { + return + } + + switch tag = buf[0]; tag { + default: + panic("internal error") + case tagFreeShort, tagFreeLong: + if atoms, fprev, fnext, err = a.verifyUnused(next, totalAtoms, tag, log, true); err != nil { + return + } + + if min := list.minSize; atoms < min { + err = &ErrILSEQ{Type: ErrFLTSize, Off: h2off(next), Arg: atoms, Arg2: min} + log(err) + return + } + + if fprev != prev { + err = &ErrILSEQ{Type: ErrFreeChaining, Off: h2off(next)} + log(err) + return + } + } + } + + } + + if bits == 0 { // Verify succeeded + if stats != nil { + *stats = st + } + return + } + + // Phase 4 - if after phase 3 there are lost free blocks, report all of + // them to 'log' + for i := range ubuf { // setup zeros for compares + ubuf[i] = 0 + } + + var off, lh int64 + rem, err := bitmap.Size() + if err != nil { + return err + } + + for rem != 0 { + rq := int(mathutil.MinInt64(64*1024, rem)) + var n int + if n, err = bitmap.ReadAt(buf[:rq], off); n != rq { + return &ErrILSEQ{Type: ErrOther, Off: off, More: fmt.Errorf("bitmap ReadAt(size %d, off %#x): %s", rq, off, err)} + } + + if !bytes.Equal(buf[:rq], ubuf[:rq]) { + for d, v := range buf[:rq] { + if v != 0 { + for i, m := range bitMask { + if v&m != 0 { + lh = 8*(off+int64(d)) + int64(i) + err = &ErrILSEQ{Type: ErrLostFreeBlock, Off: h2off(lh)} + log(err) + return + } + } + } + } + } + + off += int64(rq) + rem -= int64(rq) + } + + return +} + +type fltSlot struct { + head int64 + minSize int64 +} + +func (f fltSlot) String() string { + return fmt.Sprintf("head %#x, minSize %#x\n", f.head, f.minSize) +} + +type flt [14]fltSlot + +func (f *flt) init() { + sz := 1 + for i := range *f { + f[i].minSize, f[i].head = int64(sz), 0 + sz <<= 1 + } + f[13].minSize = 4112 +} + +func (f *flt) load(fi Filer, off int64) (err error) { + b := bufs.GCache.Get(fltSz) + defer bufs.GCache.Put(b) + if _, err = fi.ReadAt(b[:], off); err != nil { + return + } + + for i := range *f { + off := 8*i + 1 + f[i].head = b2h(b[off:]) + } + return +} + +func (f *flt) find(rq int) (h int64) { + switch { + case rq < 1: + panic(rq) + case rq >= maxFLTRq: + h, f[13].head = f[13].head, 0 + return + default: + g := f[mathutil.Log2Uint16(uint16(rq)):] + for i := range g { + p := &g[i] + if rq <= int(p.minSize) { + if h = p.head; h != 0 { + p.head = 0 + return + } + } + } + return + } +} + +func (f *flt) head(atoms int64) (h int64) { + switch { + case atoms < 1: + panic(atoms) + case atoms >= maxFLTRq: + return f[13].head + default: + lg := mathutil.Log2Uint16(uint16(atoms)) + g := f[lg:] + for i := range g { + if atoms < g[i+1].minSize { + return g[i].head + } + } + panic("internal error") + } +} + +func (f *flt) setHead(h, atoms int64, fi Filer) (err error) { + switch { + case atoms < 1: + panic(atoms) + case atoms >= maxFLTRq: + b := bufs.GCache.Get(7) + defer bufs.GCache.Put(b) + if _, err = fi.WriteAt(h2b(b[:], h), 8*13+1); err != nil { + return + } + + f[13].head = h + return + default: + lg := mathutil.Log2Uint16(uint16(atoms)) + g := f[lg:] + for i := range f { + if atoms < g[i+1].minSize { + b := bufs.GCache.Get(7) + defer bufs.GCache.Put(b) + if _, err = fi.WriteAt(h2b(b[:], h), 8*int64(i+lg)+1); err != nil { + return + } + + g[i].head = h + return + } + } + panic("internal error") + } +} + +func (f *flt) String() string { + a := []string{} + for i, v := range *f { + a = append(a, fmt.Sprintf("[%2d] %s", i, v)) + } + return strings.Join(a, "") +} + +type node struct { + b []byte + h int64 + prev, next *node +} + +type cache []*node + +func (c *cache) get(n int) *node { + r, _ := c.get2(n) + return r +} + +func (c *cache) get2(n int) (r *node, isZeroed bool) { + s := *c + lens := len(s) + if lens == 0 { + return &node{b: make([]byte, n, mathutil.Min(2*n, maxBuf))}, true + } + + i := sort.Search(lens, func(x int) bool { return len(s[x].b) >= n }) + if i == lens { + i-- + s[i].b, isZeroed = make([]byte, n, mathutil.Min(2*n, maxBuf)), true + } + + r = s[i] + r.b = r.b[:n] + copy(s[i:], s[i+1:]) + s = s[:lens-1] + *c = s + return +} + +func (c *cache) cget(n int) (r *node) { + r, ok := c.get2(n) + if ok { + return + } + + for i := range r.b { + r.b[i] = 0 + } + return +} + +func (c *cache) size() (sz int64) { + for _, n := range *c { + sz += int64(cap(n.b)) + } + return +} + +func (c *cache) put(n *node) *node { + s := *c + n.b = n.b[:cap(n.b)] + lenb := len(n.b) + lens := len(s) + i := sort.Search(lens, func(x int) bool { return len(s[x].b) >= lenb }) + s = append(s, nil) + copy(s[i+1:], s[i:]) + s[i] = n + *c = s + return n +} + +type lst struct { + front, back *node +} + +func (l *lst) pushFront(n *node) *node { + if l.front == nil { + l.front, l.back, n.prev, n.next = n, n, nil, nil + return n + } + + n.prev, n.next, l.front.prev, l.front = nil, l.front, n, n + return n +} + +func (l *lst) remove(n *node) *node { + if n.prev == nil { + l.front = n.next + } else { + n.prev.next = n.next + } + if n.next == nil { + l.back = n.prev + } else { + n.next.prev = n.prev + } + n.prev, n.next = nil, nil + return n +} + +func (l *lst) removeBack() *node { + return l.remove(l.back) +} + +func (l *lst) moveToFront(n *node) *node { + return l.pushFront(l.remove(n)) +} + +func (l *lst) size() (sz int64) { + for n := l.front; n != nil; n = n.next { + sz += int64(cap(n.b)) + } + return +} + +func cacheAudit(m map[int64]*node, l *lst) (err error) { + cnt := 0 + for h, n := range m { + if g, e := n.h, h; g != e { + return fmt.Errorf("cacheAudit: invalid node handle %d != %d", g, e) + } + + if cnt, err = l.audit(n, true); err != nil { + return + } + } + + if g, e := cnt, len(m); g != e { + return fmt.Errorf("cacheAudit: invalid cache size %d != %d", g, e) + } + + return +} + +func (l *lst) audit(n *node, onList bool) (cnt int, err error) { + if !onList && (n.prev != nil || n.next != nil) { + return -1, fmt.Errorf("lst.audit: free node with non nil linkage") + } + + if l.front == nil && l.back != nil || l.back == nil && l.front != nil { + return -1, fmt.Errorf("lst.audit: one of .front/.back is nil while the other is non nil") + } + + if l.front == l.back && l.front != nil { + x := l.front + if x.prev != nil || x.next != nil { + return -1, fmt.Errorf("lst.audit: single node has non nil linkage") + } + + if onList && x != n { + return -1, fmt.Errorf("lst.audit: single node is alien") + } + } + + seen := false + var prev *node + x := l.front + for x != nil { + cnt++ + if x.prev != prev { + return -1, fmt.Errorf("lst.audit: broken .prev linkage") + } + + if x == n { + seen = true + } + + prev = x + x = x.next + } + + if prev != l.back { + return -1, fmt.Errorf("lst.audit: broken .back linkage") + } + + if onList && !seen { + return -1, fmt.Errorf("lst.audit: node missing in list") + } + + if !onList && seen { + return -1, fmt.Errorf("lst.audit: node should not be on the list") + } + + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/falloc_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/falloc_test.go new file mode 100644 index 00000000000..f4f9ba21f25 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/falloc_test.go @@ -0,0 +1,1833 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "bytes" + "encoding/hex" + "flag" + "fmt" + "math/rand" + "os" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/cznic/bufs" + "github.com/cznic/sortutil" + "github.com/cznic/zappy" +) + +var ( + allocRndTestLimit = flag.Uint("lim", 2*maxShort, "Allocator rnd test initial blocks size limit") + allocRndTestHardLimit = flag.Uint("hlim", 0, "Allocator rnd test initial blocks size hard limit") + testN = flag.Int("N", 128, "Allocator rnd test block count") + allocRndDump = flag.Bool("dump", false, "Produce dump files on TestAllocatorRnd crash") + oKeep = flag.Bool("keep", false, "do not delete testing DB/WAL (where applicable)") +) + +func init() { + reallocTestHook = true +} + +func mfBytes(f Filer) []byte { + var b bytes.Buffer + if _, err := f.(*MemFiler).WriteTo(&b); err != nil { + panic(err) + } + + return b.Bytes() +} + +// Paranoid Allocator, automatically verifies whenever possible. +type pAllocator struct { + *Allocator + errors []error + logger func(error) bool + lastKnownGood *MemFiler + lastKnownGoodFLT flt + lastOp string + stats AllocStats +} + +func newPAllocator(f Filer) (*pAllocator, error) { + a, err := NewAllocator(f, &Options{}) + if err != nil { + return nil, err + } + + r := &pAllocator{Allocator: a, lastKnownGood: NewMemFiler()} + r.logger = func(err error) bool { + r.errors = append(r.errors, err) + return len(r.errors) < 100 + } + + return r, nil +} + +func (a *pAllocator) err() error { + var n int + if n = len(a.errors); n == 0 { + return nil + } + + s := make([]string, n) + for i, e := range a.errors { + s[i] = e.Error() + } + return fmt.Errorf("\n%s", strings.Join(s, "\n")) +} + +func (a *pAllocator) preMortem(s string) { + var e error + if e := a.lastKnownGood.Truncate(0); e != nil { + panic(e) + } + b := mfBytes(a.Allocator.f) + if _, e = a.lastKnownGood.WriteAt(b, 0); e != nil { + return + } + a.lastKnownGoodFLT = a.flt + a.lastOp = s +} + +func (a *pAllocator) Alloc(b []byte) (handle int64, err error) { + if *allocRndDump { + a.preMortem("") + defer func() { a.lastOp = fmt.Sprintf("Alloc(%d bytes): h %#x", len(b), handle) }() + } + + if handle, err = a.Allocator.Alloc(b); err != nil { + return + } + + if err = a.Allocator.Verify(NewMemFiler(), a.logger, &a.stats); err != nil { + err = fmt.Errorf("'%s': %v", err, a.err()) + return + } + + err = a.err() + return +} + +func (a *pAllocator) Free(handle int64) (err error) { + if *allocRndDump { + a.preMortem(fmt.Sprintf("Free(h %#x)", handle)) + } + + if err = a.Allocator.Free(handle); err != nil { + return + } + + if err = a.Allocator.Verify(NewMemFiler(), a.logger, &a.stats); err != nil { + err = fmt.Errorf("'%s': %v", err, a.err()) + return + } + + err = a.err() + return +} + +func (a *pAllocator) Realloc(handle int64, b []byte) (err error) { + if *allocRndDump { + a.preMortem(fmt.Sprintf("Realloc(h %#x, %d bytes)", handle, len(b))) + } + + if err = a.Allocator.Realloc(handle, b); err != nil { + return + } + + if err = cacheAudit(a.Allocator.m, &a.Allocator.lru); err != nil { + return + } + + if err = a.Allocator.Verify(NewMemFiler(), a.logger, &a.stats); err != nil { + err = fmt.Errorf("'%s': %v", err, a.err()) + return + } + + err = a.err() + return +} + +func dump(a *pAllocator, t *testing.T) { + m := a.f.(*MemFiler) + sz, err := m.Size() + if err != nil { + t.Fatal(err) + } + + t.Logf("MemFiler.Size() == %d(%#x)", sz, sz) + if !*allocRndDump { + return + } + + fn := "good-dump" + f, err := os.Create(fn) + if err != nil { + t.Fatal(err) + } + + defer f.Close() + sz, err = a.lastKnownGood.WriteTo(f) + if err != nil { + t.Error(err) + return + } + + t.Logf("%d(%#x) writen to %q", sz, sz, fn) + + fn = "bad-dump" + g, err := os.Create(fn) + if err != nil { + t.Fatal(err) + } + + defer g.Close() + sz, err = m.WriteTo(g) + if err != nil { + t.Error(err) + return + } + + t.Logf("%d(%#x) writen to %q", sz, sz, fn) + + t.Log("Last known good FLT") + for _, slot := range a.lastKnownGoodFLT { + if h := slot.head; h != 0 { + t.Logf("min %d head %#x off %#x", slot.minSize, h, h2off(h)) + } + } + + t.Log("Current FLT") + r := a.flt + for _, slot := range r { + if h := slot.head; h != 0 { + t.Logf("min %d head %#x off %#x", slot.minSize, h, h2off(h)) + } + } + t.Logf("Last op: %q", a.lastOp) +} + +func init() { + if *testN <= 0 { + *testN = 1 + } +} + +func TestVerify0(t *testing.T) { + // All must fail + tab := []string{ + + // 0: Reloc, links beyond EOF + "" + + "fd 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00", + // 1: Reloc, links beyond EOF + "" + + "fd 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 2: Reloc, broken target + "" + + "fd 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 3: Free block at file tail + "" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe", + // 4: Free block at file tail + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe", + // 5: Reloc, invalid target 0xfe + "" + + "fd 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 6: Reloc, invalid target 0xfd + "" + + "fd 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "fd 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 7: Lost free block @ 0x00 + "" + + "fe 00 00 00 00 00 00 02 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 8: Lost free block @ 0x10 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 02 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 9: Invalid padding + "" + + "00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 10: Invalid padding + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00", + // 11: Invalid padding + "" + + "01 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 12: Invalid padding + "" + + "01 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00", + // 13: Invalid padding + "" + + "0d 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00", + // 14: Invalid CC (tail tag) + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02", + // 15: Invalid CC (tail tag) + "" + + "fd 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01", + // 16: Cannot decompress + "" + + "0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01", + // 17: Invalid reloc target + "" + + "fd 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 18: Invalid tail tag @1 + "" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 19: Invalid size @1 + "" + + "ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 20: Invalid size @1 + "" + + "ff 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 21: Invalid size @1 + "" + + "ff 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 22: Invalid .next @1 + "" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 04 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 23: Invalid .prev @1 + "" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 04 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 24: Invalid tail tag @1 + "" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 25: Invalid tail size @1 + "" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 26: Invalid tail size @1 + "" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 27: Invalid tail size @1 + "" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f + } + + for i, test := range tab { + errors := []error{} + + f := NewMemFiler() + b := s2b(test) + b = append(make([]byte, fltSz), b...) + n := len(b) + if n == 0 { + t.Fatal(n) + } + + if m, err := f.ReadFrom(bytes.NewBuffer(b)); m != int64(n) || err != nil { + t.Fatal(m, err) + } + + sz, err := f.Size() + if err != nil { + t.Fatal(err) + } + + if g, e := sz, int64(n); g != e { + t.Fatal(g, e) + } + + a, err := newPAllocator(f) + if err != nil { + t.Fatal(err) + } + + err = a.Verify( + NewMemFiler(), + func(err error) bool { + if err == nil { + t.Fatal("nil error") + } + errors = append(errors, err) + return false + }, + nil, + ) + if err == nil { + t.Fatal(i, "unexpected success") + } + + t.Log(i, err, errors) + } +} + +func TestVerify1(t *testing.T) { + f := NewMemFiler() + bitmap := NewMemFiler() + if n, err := bitmap.WriteAt([]byte{0}, 0); n != 1 || err != nil { + t.Fatal(n, err) + } + + a, err := newPAllocator(f) + if err != nil { + t.Fatal(err) + } + + if err := a.Verify( + bitmap, + func(error) bool { + panic("intrnal error") + }, + nil, + ); err == nil { + t.Fatal("unexpected success") + } +} + +func repDump(a flt) string { + b := []string{} + for _, v := range a { + if h := v.head; h != 0 { + b = append(b, fmt.Sprintf("min:%d, h:%d", v.minSize, h)) + } + } + return strings.Join(b, ";") +} + +func TestVerify2(t *testing.T) { + // All must fail for the fixed (see bellow) FLT.Report() + tab := []string{ + + // 0: FLT broken linkage (missing free blocks @2,4) + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 1: FLT broken linkage (missing free block @4) + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 2: bad size @4 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // // 3: bad size @4 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 04 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // // 4: bad .next @6 from @2 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 06 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // // 5: bad .prev @7 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 07 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // // 6: bad .next @7 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 07 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 02 00 00 00 00 00 00 07 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // // 7: bad .next @5 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 07 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 02 00 00 00 00 00 00 01 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // // 8: bad chaining + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 07 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 02 00 00 00 00 00 00 01 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // // 9: lost free block @8 + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 0f fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "ff 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ff" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00" + + "fe 00 00 00 00 00 00 02 00 00 00 00 00 00 00 fe" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + // 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f + } + + for i, test := range tab { + errors := []error{} + + f := NewMemFiler() + b := s2b(test) + b = append(make([]byte, fltSz), b...) + n := len(b) + if n == 0 { + t.Fatal(n) + } + + if m, err := f.ReadFrom(bytes.NewBuffer(b)); m != int64(n) || err != nil { + t.Fatal(m, err) + } + + sz, err := f.Size() + if err != nil { + t.Fatal(err) + } + + if g, e := sz, int64(n); g != e { + t.Fatal(g, e) + } + + a, err := newPAllocator(f) + if err != nil { + t.Fatal(err) + } + + a.flt.setHead(2, 1, a.f) + a.flt.setHead(4, 2, a.f) + err = a.Verify( + NewMemFiler(), + func(err error) bool { + if err == nil { + t.Fatal("nil error") + } + t.Log(i, "logged: ", err) + errors = append(errors, err) + return true + }, + nil, + ) + if err == nil { + t.Fatal(i, "unexpected success") + } + + t.Log(i, err, errors) + } +} + +// Allocation in an empty DB. +func TestAllocatorAlloc0(t *testing.T) { + tab := []struct { + h int64 + b, f, fc string + }{ + {1, // len 0 + "" + + "", + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + "" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"}, + {1, // len 1 + "" + + "42", + "" + + "01 42 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + "" + + "01 42 00 00 00 00 00 00 00 00 00 00 00 00 00 00"}, + {1, // max single atom, not compressible + "" + + "01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e", + "" + + "0e 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 00", + "" + + "0e 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 00"}, + {1, // max single atom, compressible, but not eligible for it + "" + + "01 02 03 04 05 06 07 08 99 01 02 03 04 05", + "" + + "0e 01 02 03 04 05 06 07 08 99 01 02 03 04 05 00", + "" + + "0e 01 02 03 04 05 06 07 08 99 01 02 03 04 05 00"}, + {1, // > 1 atom, not compressible + "" + + "01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f", + "" + + "0f 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + "" + + "0f 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"}, + {1, // > 1 atom, compressible + "" + + "01 02 03 04 05 06 07 08 99 01 02 03 04 05 06 07" + + "08", + "" + + "11 01 02 03 04 05 06 07 08 99 01 02 03 04 05 06" + + "07 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + "" + + "0e 11 12 01 02 03 04 05 06 07 08 99 01 0d 09 01"}, + {1, // longest short + "" + + "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "10 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "20 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "30 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "40 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "50 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "60 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "70 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "80 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "90 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "a0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "b0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "c0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "d0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "e0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "f0 01 02 03 04 05 06 07 08 09 0a", + "" + + "" + + "fb 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 10 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 20 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 30 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 40 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 50 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 60 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 70 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 80 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f 90 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f a0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f b0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f c0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f d0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f e0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e" + + "0f f0 01 02 03 04 05 06 07 08 09 0a 00 00 00 00", + "" + + "" + + "4e fb 01 20 00 01 02 03 04 05 06 07 08 09 0a 0b" + + "0c 0d 0e 0f 10 1d 10 00 20 1d 10 00 30 1d 10 00" + + "40 1d 10 00 50 1d 10 00 60 1d 10 00 70 1d 10 00" + + "80 1d 10 00 90 1d 10 00 a0 1d 10 00 b0 1d 10 00" + + "c0 1d 10 00 d0 1d 10 00 e0 1d 10 00 f0 13 10 01"}, + + {1, // shortest long + "" + + "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "10 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "20 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "30 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "40 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "50 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "60 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "70 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "80 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "90 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "a0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "b0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "c0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "d0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "e0 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" + + "f0 01 02 03 04 05 06 07 08 09 0a 0b", + "" + + "" + + "fc 00 fc 00 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 10 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 20 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 30 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 40 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 50 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 60 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 70 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 80 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f 90 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f a0 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f b0 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f c0 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f d0 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f e0 01 02 03 04 05 06 07 08 09 0a 0b 0c" + + "0d 0e 0f f0 01 02 03 04 05 06 07 08 09 0a 0b 00", + "" + + "" + + "4e fc 01 20 00 01 02 03 04 05 06 07 08 09 0a 0b" + + "0c 0d 0e 0f 10 1d 10 00 20 1d 10 00 30 1d 10 00" + + "40 1d 10 00 50 1d 10 00 60 1d 10 00 70 1d 10 00" + + "80 1d 10 00 90 1d 10 00 a0 1d 10 00 b0 1d 10 00" + + "c0 1d 10 00 d0 1d 10 00 e0 1d 10 00 f0 15 10 01"}, + } + + for i, test := range tab { + f := func(compress bool, e []byte) { + f := NewMemFiler() + a, err := newPAllocator(f) + if err != nil { + t.Fatal(err) + } + + a.Compress = compress + h, err := a.Alloc(s2b(test.b)) + if err != nil { + t.Fatalf("%d %#v\n%s", i, err, hex.Dump(mfBytes(f))) + } + + if g, e := h, test.h; g != e { + t.Fatal(i, g, e) + } + + g := mfBytes(f) + if g = g[fltSz:]; !bytes.Equal(g, e) { + t.Fatalf("\ni: %d compress: %t\ng:\n%se:\n%s", i, compress, hex.Dump(g), hex.Dump(e)) + } + } + f(false, s2b(test.f)) + f(true, s2b(test.fc)) + } +} + +func TestAllocatorMakeUsedBlock(t *testing.T) { + f := NewMemFiler() + a, err := NewAllocator(f, &Options{}) + if err != nil { + t.Fatal(err) + } + + dst := bufs.GCache.Get(zappy.MaxEncodedLen(maxRq + 1)) + defer bufs.GCache.Put(dst) + if _, _, _, err := a.makeUsedBlock(dst, make([]byte, maxRq)); err != nil { + t.Fatal(err) + } + + if _, _, _, err := a.makeUsedBlock(dst, make([]byte, maxRq+1)); err == nil { + t.Fatal("unexpected success") + } +} + +func stableRef(m map[int64][]byte) (r []struct { + h int64 + b []byte +}) { + a := make(sortutil.Int64Slice, 0, len(m)) + for k := range m { + a = append(a, k) + } + sort.Sort(a) + for _, v := range a { + r = append(r, struct { + h int64 + b []byte + }{v, m[v]}) + } + return +} + +func TestAllocatorRnd(t *testing.T) { + N := *testN + + for cc := 0; cc < 2; cc++ { + rng := rand.New(rand.NewSource(42)) + f := NewMemFiler() + a, err := newPAllocator(f) + if err != nil { + t.Fatal(err) + } + + balance := 0 + + bad := func() bool { + if a.Compress { + return false + } + + actual := a.stats.TotalAtoms - a.stats.FreeAtoms - a.stats.Relocations + if int64(balance) != actual { + t.Logf("balance: %d, actual %d\n%#v", balance, actual, a.stats) + return true + } + + return false + } + + if cc != 0 { + a.Compress = true + } + ref := map[int64][]byte{} + + for pass := 0; pass < 2; pass++ { + + // A) Alloc N blocks + for i := 0; i < N; i++ { + rq := rng.Int31n(int32(*allocRndTestLimit)) + if rq%127 == 0 { + rq = 3 * maxRq / 4 + } + if rq%11 == 0 { + rq %= 23 + } + if hl := *allocRndTestHardLimit; hl != 0 { + rq = rq % int32(hl) + } + b := make([]byte, rq) + for j := range b { + b[j] = byte(rng.Int()) + } + if rq > 300 { + for i := 100; i < 200; i++ { + b[i] = 'A' // give compression a chance + } + } + + balance += n2atoms(len(b)) + h, err := a.Alloc(b) + if err != nil || bad() { + dump(a, t) + t.Fatalf( + "A) N %d, kind %d, pass %d, i:%d, len(b):%d(%#x), err %v", + N, 0, pass, i, len(b), len(b), err, + ) + } + + ref[h] = b + } + + var rb []byte + + // B) Check them back + for h, wb := range ref { + if rb, err = a.Get(rb, h); err != nil { + dump(a, t) + t.Fatal("B)", err) + } + + if !bytes.Equal(rb, wb) { + dump(a, t) + t.Fatalf("B) h %d", h) + } + } + + nf := 0 + // C) Free every third block + for _, v := range stableRef(ref) { + h, b := v.h, v.b + if rng.Int()%3 != 0 { + continue + } + + balance -= n2atoms(len(b)) + if err = a.Free(h); err != nil || bad() { + dump(a, t) + t.Fatal(err) + } + + delete(ref, h) + nf++ + } + + // D) Check them back + for h, wb := range ref { + if rb, err = a.Get(rb, h); err != nil { + dump(a, t) + t.Fatal("D)", err) + } + + if !bytes.Equal(rb, wb) { + dump(a, t) + t.Fatalf("D) h %d", h) + } + } + + // E) Resize every block remaining + for _, v := range stableRef(ref) { + h, wb := v.h, append([]byte(nil), v.b...) + len0 := len(wb) + switch rng.Int() & 1 { + case 0: + wb = wb[:len(wb)*3/4] + case 1: + wb = append(wb, wb...) + } + if len(wb) > maxRq { + wb = wb[:maxRq] + } + + for j := range wb { + wb[j] = byte(rng.Int()) + } + if len(wb) > 300 { + for i := 100; i < 200; i++ { + wb[i] = 'D' // give compression a chance + } + } + a0, a1 := n2atoms(len0), n2atoms(len(wb)) + balance = balance - a0 + a1 + if err := a.Realloc(h, wb); err != nil || bad() { + dump(a, t) + t.Fatalf( + "D) h:%#x, len(b):%#4x, len(wb): %#x, err %v", + h, len0, len(wb), err, + ) + } + + if err = cacheAudit(a.m, &a.lru); err != nil { + t.Fatal(err) + } + + ref[h] = wb + } + + // F) Check them back + for h, wb := range ref { + if rb, err = a.Get(rb, h); err != nil { + dump(a, t) + t.Fatal("E)", err) + } + + if !bytes.Equal(rb, wb) { + dump(a, t) + t.Fatalf("E) h %d", h) + } + } + } + + if cc == 0 { + sz, err := f.Size() + if err != nil { + t.Fatal(err) + } + + t.Logf( + "kind %d, AllocAtoms %7d, AllocBytes %7d, FreeAtoms %7d, Relocations %7d, TotalAtoms %7d, f.Size %7d, space eff %.2f%%", + 0, a.stats.AllocAtoms, a.stats.AllocBytes, a.stats.FreeAtoms, a.stats.Relocations, a.stats.TotalAtoms, sz, 100*float64(a.stats.AllocBytes)/float64(sz), + ) + } + // Free everything + for h, b := range ref { + balance -= n2atoms(len(b)) + if err = a.Free(h); err != nil || bad() { + dump(a, t) + t.Fatal(err) + } + } + + sz, err := a.f.Size() + if err != nil { + t.Fatal(err) + } + + if g, e := sz, int64(fltSz); g != e { + dump(a, t) + t.Fatal(g, e) + } + } +} + +func TestRollbackAllocator(t *testing.T) { + f := NewMemFiler() + var r *RollbackFiler + r, err := NewRollbackFiler(f, + func(sz int64) (err error) { + if err = f.Truncate(sz); err != nil { + return err + } + + return f.Sync() + }, + f, + ) + if err != nil { + t.Fatal(err) + } + + if err := r.BeginUpdate(); err != nil { // BeginUpdate 0->1 + t.Fatal(err) + } + + a, err := NewAllocator(r, &Options{}) + if err != nil { + t.Fatal(err) + } + + h, err := a.Alloc(nil) + if err != nil { + t.Fatal(err) + } + + if h != 1 { + t.Fatal(h) + } + + // | 1 | + + h, err = a.Alloc(nil) + if err != nil { + t.Fatal(err) + } + + if h != 2 { + t.Fatal(h) + } + + // | 1 | 2 | + h, err = a.Alloc(nil) + if err != nil { + t.Fatal(err) + } + + if h != 3 { + t.Fatal(h) + } + + // | 1 | 2 | 3 | + if err = a.Free(2); err != nil { + t.Fatal(err) + } + + // | 1 | free | 3 | + if err := r.BeginUpdate(); err != nil { // BeginUpdate 1->2 + t.Fatal(err) + } + + h, err = a.Alloc(nil) + if err != nil { + t.Fatal(err) + } + + if h != 2 { + t.Fatal(h) + } + + // | 1 | 2 | 3 | + if err := r.Rollback(); err != nil { // Rollback 2->1 + t.Fatal(err) + } + + // | 1 | free | 3 | + h, err = a.Alloc(nil) + if err != nil { + t.Fatal(err) + } + + if h != 2 { + t.Fatal(h) + } + + // | 1 | 2 | 3 | + if err := a.Verify(NewMemFiler(), nil, nil); err != nil { + t.Fatal(err) + } +} + +func benchmarkAllocatorAlloc(b *testing.B, f Filer, sz int) { + if err := f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + a, err := NewAllocator(f, &Options{}) + if err != nil { + b.Error(err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + + v := make([]byte, sz) + runtime.GC() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err = f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + if h, err := a.Alloc(v); h <= 0 || err != nil { + f.EndUpdate() + b.Error(h, err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + } +} + +func benchmarkAllocatorAllocMemFiler(b *testing.B, sz int) { + f := NewMemFiler() + benchmarkAllocatorAlloc(b, f, sz) +} + +func BenchmarkAllocatorAllocMemFiler1e0(b *testing.B) { + benchmarkAllocatorAllocMemFiler(b, 0) +} + +func BenchmarkAllocatorAllocMemFiler1e1(b *testing.B) { + benchmarkAllocatorAllocMemFiler(b, 1e1) +} + +func BenchmarkAllocatorAllocMemFiler1e2(b *testing.B) { + benchmarkAllocatorAllocMemFiler(b, 1e2) +} + +func BenchmarkAllocatorAllocMemFiler1e3(b *testing.B) { + benchmarkAllocatorAllocMemFiler(b, 1e3) +} + +func benchmarkAllocatorAllocSimpleFileFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + benchmarkAllocatorAlloc(b, NewSimpleFileFiler(f), sz) +} + +func BenchmarkAllocatorAllocSimpleFileFiler0(b *testing.B) { + benchmarkAllocatorAllocSimpleFileFiler(b, 0) +} + +func BenchmarkAllocatorAllocSimpleFileFiler1e1(b *testing.B) { + benchmarkAllocatorAllocSimpleFileFiler(b, 1e1) +} + +func BenchmarkAllocatorAllocSimpleFileFiler1e2(b *testing.B) { + benchmarkAllocatorAllocSimpleFileFiler(b, 1e2) +} + +func BenchmarkAllocatorAllocSimpleFileFiler1e3(b *testing.B) { + benchmarkAllocatorAllocSimpleFileFiler(b, 1e3) +} + +func benchmarkAllocatorAllocRollbackFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + g := NewSimpleFileFiler(f) + var filer *RollbackFiler + if filer, err = NewRollbackFiler( + g, + func(sz int64) error { + if err = g.Truncate(sz); err != nil { + return err + } + + return g.Sync() + }, + g, + ); err != nil { + b.Error(err) + return + } + + benchmarkAllocatorAlloc(b, filer, sz) +} + +func BenchmarkAllocatorAllocRollbackFiler0(b *testing.B) { + benchmarkAllocatorAllocRollbackFiler(b, 0) +} + +func BenchmarkAllocatorAllocRollbackFiler1e1(b *testing.B) { + benchmarkAllocatorAllocRollbackFiler(b, 1e1) +} + +func BenchmarkAllocatorAllocRollbackFiler1e2(b *testing.B) { + benchmarkAllocatorAllocRollbackFiler(b, 1e2) +} + +func BenchmarkAllocatorAllocRollbackFiler1e3(b *testing.B) { + benchmarkAllocatorAllocRollbackFiler(b, 1e3) +} + +func benchmarkAllocatorAllocACIDFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + wal, err := os.OpenFile(testDbName+".wal", os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer wal.Close() + + filer, err := NewACIDFiler(NewSimpleFileFiler(f), wal) + if err != nil { + b.Error(err) + return + } + + benchmarkAllocatorAlloc(b, filer, sz) +} + +func BenchmarkAllocatorAllocACIDFiler0(b *testing.B) { + benchmarkAllocatorAllocACIDFiler(b, 0) +} + +func BenchmarkAllocatorAllocACIDFiler1e1(b *testing.B) { + benchmarkAllocatorAllocACIDFiler(b, 1e1) +} + +func BenchmarkAllocatorAllocACIDFiler1e2(b *testing.B) { + benchmarkAllocatorAllocACIDFiler(b, 1e2) +} + +func BenchmarkAllocatorAllocACIDFiler1e3(b *testing.B) { + benchmarkAllocatorAllocACIDFiler(b, 1e3) +} + +func benchmarkAllocatorRndFree(b *testing.B, f Filer, sz int) { + if err := f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + a, err := NewAllocator(f, &Options{}) + if err != nil { + b.Error(err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + + v := make([]byte, sz) + ref := map[int64]struct{}{} + for i := 0; i < b.N; i++ { + if err = f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + h, err := a.Alloc(v) + if h <= 0 || err != nil { + f.EndUpdate() + b.Error(h, err) + return + } + + ref[h] = struct{}{} + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + } + runtime.GC() + b.ResetTimer() + for h := range ref { + if err = f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + if err = a.Free(h); err != nil { + f.EndUpdate() + b.Error(h, err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + } +} + +func benchmarkAllocatorRndFreeMemFiler(b *testing.B, sz int) { + f := NewMemFiler() + benchmarkAllocatorRndFree(b, f, sz) +} + +func BenchmarkAllocatorRndFreeMemFiler0(b *testing.B) { + benchmarkAllocatorRndFreeMemFiler(b, 0) +} + +func BenchmarkAllocatorRndFreeMemFiler1e1(b *testing.B) { + benchmarkAllocatorRndFreeMemFiler(b, 1e1) +} + +func BenchmarkAllocatorRndFreeMemFiler1e2(b *testing.B) { + benchmarkAllocatorRndFreeMemFiler(b, 1e2) +} + +func BenchmarkAllocatorRndFreeMemFiler1e3(b *testing.B) { + benchmarkAllocatorRndFreeMemFiler(b, 1e3) +} + +func benchmarkAllocatorRndFreeSimpleFileFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + benchmarkAllocatorRndFree(b, NewSimpleFileFiler(f), sz) +} + +func BenchmarkAllocatorRndFreeSimpleFileFiler0(b *testing.B) { + benchmarkAllocatorRndFreeSimpleFileFiler(b, 0) +} + +func BenchmarkAllocatorRndFreeSimpleFileFiler1e1(b *testing.B) { + benchmarkAllocatorRndFreeSimpleFileFiler(b, 1e1) +} + +func BenchmarkAllocatorRndFreeSimpleFileFiler1e2(b *testing.B) { + benchmarkAllocatorRndFreeSimpleFileFiler(b, 1e2) +} + +func BenchmarkAllocatorRndFreeSimpleFileFiler1e3(b *testing.B) { + benchmarkAllocatorRndFreeSimpleFileFiler(b, 1e3) +} + +func benchmarkAllocatorRndFreeRollbackFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + g := NewSimpleFileFiler(f) + var filer *RollbackFiler + if filer, err = NewRollbackFiler( + g, + func(sz int64) error { + if err = g.Truncate(sz); err != nil { + return err + } + + return g.Sync() + }, + g, + ); err != nil { + b.Error(err) + return + } + + benchmarkAllocatorRndFree(b, filer, sz) +} + +func BenchmarkAllocatorRndFreeRollbackFiler0(b *testing.B) { + benchmarkAllocatorRndFreeRollbackFiler(b, 0) +} + +func BenchmarkAllocatorRndFreeRollbackFiler1e1(b *testing.B) { + benchmarkAllocatorRndFreeRollbackFiler(b, 1e1) +} + +func BenchmarkAllocatorRndFreeRollbackFiler1e2(b *testing.B) { + benchmarkAllocatorRndFreeRollbackFiler(b, 1e2) +} + +func BenchmarkAllocatorRndFreeRollbackFiler1e3(b *testing.B) { + benchmarkAllocatorRndFreeRollbackFiler(b, 1e3) +} + +func benchmarkAllocatorRndFreeACIDFiler(b *testing.B, sz int) { + dir, testDbName := temp() + defer os.RemoveAll(dir) + + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer f.Close() + + wal, err := os.OpenFile(testDbName+".wal", os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer wal.Close() + + filer, err := NewACIDFiler(NewSimpleFileFiler(f), wal) + if err != nil { + b.Error(err) + return + } + + benchmarkAllocatorRndFree(b, filer, sz) +} + +func BenchmarkAllocatorRndFreeACIDFiler0(b *testing.B) { + benchmarkAllocatorRndFreeACIDFiler(b, 0) +} + +func BenchmarkAllocatorRndFreeACIDFiler1e1(b *testing.B) { + benchmarkAllocatorRndFreeACIDFiler(b, 1e1) +} + +func BenchmarkAllocatorRndFreeACIDFiler1e2(b *testing.B) { + benchmarkAllocatorRndFreeACIDFiler(b, 1e2) +} + +func BenchmarkAllocatorRndFreeACIDFiler1e3(b *testing.B) { + benchmarkAllocatorRndFreeACIDFiler(b, 1e3) +} + +func benchmarkAllocatorRndGet(b *testing.B, f Filer, sz int) { + if err := f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + a, err := NewAllocator(f, &Options{}) + if err != nil { + b.Error(err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + + v := make([]byte, sz) + ref := map[int64]struct{}{} + if err = f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + for i := 0; i < b.N; i++ { + h, err := a.Alloc(v) + if h <= 0 || err != nil { + f.EndUpdate() + b.Error(h, err) + return + } + + ref[h] = struct{}{} + + } + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + + runtime.GC() + b.ResetTimer() + for h := range ref { + if err = f.BeginUpdate(); err != nil { + b.Error(err) + return + } + + if _, err = a.Get(v, h); err != nil { + f.EndUpdate() + b.Error(h, err) + return + } + + if err = f.EndUpdate(); err != nil { + b.Error(err) + return + } + } +} + +func benchmarkAllocatorRndGetMemFiler(b *testing.B, sz int) { + f := NewMemFiler() + benchmarkAllocatorRndGet(b, f, sz) +} + +func BenchmarkAllocatorRndGetMemFiler0(b *testing.B) { + benchmarkAllocatorRndGetMemFiler(b, 0) +} + +func BenchmarkAllocatorRndGetMemFiler1e1(b *testing.B) { + benchmarkAllocatorRndGetMemFiler(b, 1e1) +} + +func BenchmarkAllocatorRndGetMemFiler1e2(b *testing.B) { + benchmarkAllocatorRndGetMemFiler(b, 1e2) +} + +func BenchmarkAllocatorRndGetMemFiler1e3(b *testing.B) { + benchmarkAllocatorRndGetMemFiler(b, 1e3) +} + +func benchmarkAllocatorRndGetSimpleFileFiler(b *testing.B, sz int) { + os.Remove(testDbName) + <-time.After(5 * time.Second) + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer func() { + f.Close() + os.Remove(testDbName) + }() + + benchmarkAllocatorRndGet(b, NewSimpleFileFiler(f), sz) +} + +func BenchmarkAllocatorRndGetSimpleFileFiler0(b *testing.B) { + benchmarkAllocatorRndGetSimpleFileFiler(b, 0) +} + +func BenchmarkAllocatorRndGetSimpleFileFiler1e1(b *testing.B) { + benchmarkAllocatorRndGetSimpleFileFiler(b, 1e1) +} + +func BenchmarkAllocatorRndGetSimpleFileFiler1e2(b *testing.B) { + benchmarkAllocatorRndGetSimpleFileFiler(b, 1e2) +} + +func BenchmarkAllocatorRndGetSimpleFileFiler1e3(b *testing.B) { + benchmarkAllocatorRndGetSimpleFileFiler(b, 1e3) +} + +func benchmarkAllocatorRndGetRollbackFiler(b *testing.B, sz int) { + os.Remove(testDbName) + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer func() { + f.Close() + os.Remove(testDbName) + }() + + g := NewSimpleFileFiler(f) + var filer *RollbackFiler + if filer, err = NewRollbackFiler( + g, + func(sz int64) error { + if err = g.Truncate(sz); err != nil { + return err + } + + return g.Sync() + }, + g, + ); err != nil { + b.Error(err) + return + } + + benchmarkAllocatorRndGet(b, filer, sz) +} + +func BenchmarkAllocatorRndGetRollbackFiler0(b *testing.B) { + benchmarkAllocatorRndGetRollbackFiler(b, 0) +} + +func BenchmarkAllocatorRndGetRollbackFiler1e1(b *testing.B) { + benchmarkAllocatorRndGetRollbackFiler(b, 1e1) +} + +func BenchmarkAllocatorRndGetRollbackFiler1e2(b *testing.B) { + benchmarkAllocatorRndGetRollbackFiler(b, 1e2) +} + +func BenchmarkAllocatorRndGetRollbackFiler1e3(b *testing.B) { + benchmarkAllocatorRndGetRollbackFiler(b, 1e3) +} + +func benchmarkAllocatorRndGetACIDFiler(b *testing.B, sz int) { + os.Remove(testDbName) + os.Remove(walName) + f, err := os.OpenFile(testDbName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer func() { + f.Close() + os.Remove(testDbName) + }() + + wal, err := os.OpenFile(walName, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + b.Fatal(err) + } + + defer func() { + wal.Close() + os.Remove(walName) + }() + + filer, err := NewACIDFiler(NewSimpleFileFiler(f), wal) + if err != nil { + b.Error(err) + return + } + + benchmarkAllocatorRndGet(b, filer, sz) +} + +func BenchmarkAllocatorRndGetACIDFiler0(b *testing.B) { + benchmarkAllocatorRndGetACIDFiler(b, 0) +} + +func BenchmarkAllocatorRndGetACIDFiler1e1(b *testing.B) { + benchmarkAllocatorRndGetACIDFiler(b, 1e1) +} + +func BenchmarkAllocatorRndGetACIDFiler1e2(b *testing.B) { + benchmarkAllocatorRndGetACIDFiler(b, 1e2) +} + +func BenchmarkAllocatorRndGetACIDFiler1e3(b *testing.B) { + benchmarkAllocatorRndGetACIDFiler(b, 1e3) +} + +func TestFltFind(t *testing.T) { + var f flt + + f.init() + if h := f.find(1); h != 0 { + t.Fatal(h) + } + + // [0] + f.init() + f[0].head = 1 + if h := f.find(1); h != 1 || f[0].head != 0 { + t.Fatal(h) + } + + f.init() + f[0].head = 1 + if h := f.find(2); h != 0 || f[0].head == 0 { + t.Fatal(h) + } + + // [1] + f.init() + f[1].head = 1 + if h := f.find(1); h != 1 || f[1].head != 0 { + t.Fatal("\n", f, h) + } + + f.init() + f[1].head = 1 + if h := f.find(2); h != 1 || f[1].head != 0 { + t.Fatal(f, h) + } + + f.init() + f[1].head = 1 + if h := f.find(3); h != 0 || f[1].head == 0 { + t.Fatal(h) + } + + // [2] + f.init() + f[2].head = 1 + if h := f.find(1); h != 1 || f[2].head != 0 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.find(2); h != 1 || f[2].head != 0 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.find(3); h != 1 || f[2].head != 0 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.find(4); h != 1 || f[2].head != 0 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.find(5); h != 0 || f[2].head == 0 { + t.Fatal(h) + } +} + +func TestFltHead(t *testing.T) { + var f flt + f.init() + if h := f.head(1); h != 0 { + t.Fatal(h) + } + + // [0] + f.init() + f[0].head = 1 + if h := f.head(1); h != 1 { + t.Fatal(h) + } + + f.init() + f[0].head = 1 + if h := f.head(2); h != 0 { + t.Fatal(h) + } + + // [1] + f.init() + f[1].head = 1 + if h := f.head(1); h != 0 { + t.Fatal(h) + } + + f.init() + f[1].head = 1 + if h := f.head(2); h != 1 { + t.Fatal(h) + } + + f.init() + f[1].head = 1 + if h := f.head(3); h != 1 { + t.Fatal(h) + } + + f.init() + f[1].head = 1 + if h := f.head(4); h != 0 { + t.Fatal(h) + } + + // [2] + f.init() + f[2].head = 1 + if h := f.head(1); h != 0 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.head(2); h != 0 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.head(3); h != 0 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.head(4); h != 1 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.head(5); h != 1 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.head(6); h != 1 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.head(7); h != 1 { + t.Fatal(h) + } + + f.init() + f[2].head = 1 + if h := f.head(8); h != 0 { + t.Fatal(h) + } + +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/filer.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/filer.go new file mode 100644 index 00000000000..38b389387a1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/filer.go @@ -0,0 +1,192 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// An abstraction of file like (persistent) storage with optional (abstracted) +// support for structural integrity. + +package lldb + +import ( + "fmt" + + "github.com/cznic/mathutil" +) + +func doubleTrouble(first, second error) error { + return fmt.Errorf("%q. Additionally, while attempting to recover (rollback): %q", first, second) +} + +// A Filer is a []byte-like model of a file or similar entity. It may +// optionally implement support for structural transaction safety. In contrast +// to a file stream, a Filer is not sequentially accessible. ReadAt and WriteAt +// are always "addressed" by an offset and are assumed to perform atomically. +// A Filer is not safe for concurrent access, it's designed for consumption by +// the other objects in package, which should use a Filer from one goroutine +// only or via a mutex. BeginUpdate, EndUpdate and Rollback must be either all +// implemented by a Filer for structural integrity - or they should be all +// no-ops; where/if that requirement is relaxed. +// +// If a Filer wraps another Filer implementation, it usually invokes the same +// methods on the "inner" one, after some possible argument translations etc. +// If a Filer implements the structural transactions handling methods +// (BeginUpdate, EndUpdate and Rollback) as no-ops _and_ wraps another Filer: +// it then still MUST invoke those methods on the inner Filer. This is +// important for the case where a RollbackFiler exists somewhere down the +// chain. It's also important for an Allocator - to know when it must +// invalidate its FLT cache. +type Filer interface { + // BeginUpdate increments the "nesting" counter (initially zero). Every + // call to BeginUpdate must be eventually "balanced" by exactly one of + // EndUpdate or Rollback. Calls to BeginUpdate may nest. + BeginUpdate() error + + // Analogous to os.File.Close(). + Close() error + + // EndUpdate decrements the "nesting" counter. If it's zero after that + // then assume the "storage" has reached structural integrity (after a + // batch of partial updates). If a Filer implements some support for + // that (write ahead log, journal, etc.) then the appropriate actions + // are to be taken for nesting == 0. Invocation of an unbalanced + // EndUpdate is an error. + EndUpdate() error + + // Analogous to os.File.Name(). + Name() string + + // PunchHole deallocates space inside a "file" in the byte range + // starting at off and continuing for size bytes. The actual hole + // created by PunchHole may be smaller than requested. The Filer size + // (as reported by `Size()` does not change when hole punching, even + // when punching the end of a file off. In contrast to the Linux + // implementation of FALLOC_FL_PUNCH_HOLE in `fallocate`(2); a Filer is + // free not only to ignore `PunchHole()` (implement it as a nop), but + // additionally no guarantees about the content of the hole, when + // eventually read back, are required, i.e. any data, not only zeros, + // can be read from the "hole", including just anything what was left + // there - with all of the possible security problems. + PunchHole(off, size int64) error + + // As os.File.ReadAt. Note: `off` is an absolute "file pointer" + // address and cannot be negative even when a Filer is a InnerFiler. + ReadAt(b []byte, off int64) (n int, err error) + + // Rollback cancels and undoes the innermost pending update level. + // Rollback decrements the "nesting" counter. If a Filer implements + // some support for keeping structural integrity (write ahead log, + // journal, etc.) then the appropriate actions are to be taken. + // Invocation of an unbalanced Rollback is an error. + Rollback() error + + // Analogous to os.File.FileInfo().Size(). + Size() (int64, error) + + // Analogous to os.Sync(). + Sync() (err error) + + // Analogous to os.File.Truncate(). + Truncate(size int64) error + + // Analogous to os.File.WriteAt(). Note: `off` is an absolute "file + // pointer" address and cannot be negative even when a Filer is a + // InnerFiler. + WriteAt(b []byte, off int64) (n int, err error) +} + +var _ Filer = &InnerFiler{} // Ensure InnerFiler is a Filer. + +// A InnerFiler is a Filer with added addressing/size translation. +type InnerFiler struct { + outer Filer + off int64 +} + +// NewInnerFiler returns a new InnerFiler wrapped by `outer` in a way which +// adds `off` to every access. +// +// For example, considering: +// +// inner := NewInnerFiler(outer, 10) +// +// then +// +// inner.WriteAt([]byte{42}, 4) +// +// translates to +// +// outer.WriteAt([]byte{42}, 14) +// +// But an attempt to emulate +// +// outer.WriteAt([]byte{17}, 9) +// +// by +// +// inner.WriteAt([]byte{17}, -1) +// +// will fail as the `off` parameter can never be < 0. Also note that +// +// inner.Size() == outer.Size() - off, +// +// i.e. `inner` pretends no `outer` exists. Finally, after e.g. +// +// inner.Truncate(7) +// outer.Size() == 17 +// +// will be true. +func NewInnerFiler(outer Filer, off int64) *InnerFiler { return &InnerFiler{outer, off} } + +// BeginUpdate implements Filer. +func (f *InnerFiler) BeginUpdate() error { return f.outer.BeginUpdate() } + +// Close implements Filer. +func (f *InnerFiler) Close() (err error) { return f.outer.Close() } + +// EndUpdate implements Filer. +func (f *InnerFiler) EndUpdate() error { return f.outer.EndUpdate() } + +// Name implements Filer. +func (f *InnerFiler) Name() string { return f.outer.Name() } + +// PunchHole implements Filer. `off`, `size` must be >= 0. +func (f *InnerFiler) PunchHole(off, size int64) error { return f.outer.PunchHole(f.off+off, size) } + +// ReadAt implements Filer. `off` must be >= 0. +func (f *InnerFiler) ReadAt(b []byte, off int64) (n int, err error) { + if off < 0 { + return 0, &ErrINVAL{f.outer.Name() + ":ReadAt invalid off", off} + } + + return f.outer.ReadAt(b, f.off+off) +} + +// Rollback implements Filer. +func (f *InnerFiler) Rollback() error { return f.outer.Rollback() } + +// Size implements Filer. +func (f *InnerFiler) Size() (int64, error) { + sz, err := f.outer.Size() + if err != nil { + return 0, err + } + + return mathutil.MaxInt64(sz-f.off, 0), nil +} + +// Sync() implements Filer. +func (f *InnerFiler) Sync() (err error) { + return f.outer.Sync() +} + +// Truncate implements Filer. +func (f *InnerFiler) Truncate(size int64) error { return f.outer.Truncate(size + f.off) } + +// WriteAt implements Filer. `off` must be >= 0. +func (f *InnerFiler) WriteAt(b []byte, off int64) (n int, err error) { + if off < 0 { + return 0, &ErrINVAL{f.outer.Name() + ":WriteAt invalid off", off} + } + + return f.outer.WriteAt(b, f.off+off) +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/filer_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/filer_test.go new file mode 100644 index 00000000000..e2da7a187c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/filer_test.go @@ -0,0 +1,764 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "bytes" + "encoding/hex" + "io/ioutil" + "math/rand" + "os" + "runtime" + "testing" + + "github.com/cznic/fileutil" +) + +// Bench knobs. +const ( + filerTestChunkSize = 32e3 + filerTotalSize = 10e6 +) + +type newFunc func() Filer + +type testFileFiler struct { + Filer +} + +func (t *testFileFiler) Close() (err error) { + n := t.Name() + err = t.Filer.Close() + if errDel := os.Remove(n); errDel != nil && err == nil { + err = errDel + } + return +} + +var ( + newFileFiler = func() Filer { + file, err := ioutil.TempFile("", "lldb-test-file") + if err != nil { + panic(err) + } + + return &testFileFiler{NewSimpleFileFiler(file)} + } + + newOSFileFiler = func() Filer { + file, err := ioutil.TempFile("", "lldb-test-osfile") + if err != nil { + panic(err) + } + + return &testFileFiler{NewOSFiler(file)} + } + + newMemFiler = func() Filer { + return NewMemFiler() + } + + nwBitFiler = func() Filer { + f, err := newBitFiler(NewMemFiler()) + if err != nil { + panic(err) + } + + return f + } + + newRollbackFiler = func() Filer { + f := NewMemFiler() + + var r Filer + + checkpoint := func(sz int64) (err error) { + return f.Truncate(sz) + } + + r, err := NewRollbackFiler(f, checkpoint, f) + if err != nil { + panic(err) + } + + return r + } +) + +func TestFilerNesting(t *testing.T) { + testFilerNesting(t, newFileFiler) + testFilerNesting(t, newOSFileFiler) + testFilerNesting(t, newMemFiler) + testFilerNesting(t, newRollbackFiler) +} + +func testFilerNesting(t *testing.T, nf newFunc) { + // Check {Create, Close} works. + f := nf() + t.Log(f.Name()) + if err := f.Close(); err != nil { + t.Fatal(err) + } + + // Check {Create, EndUpdate} doesn't work. + f = nf() + t.Log(f.Name()) + if err := f.EndUpdate(); err == nil { + f.Close() + t.Fatal("unexpected success") + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + // Check {Create, BeginUpdate, Close} doesn't work. + f = nf() + t.Log(f.Name()) + f.BeginUpdate() + + if err := f.Close(); err == nil { + t.Fatal("unexpected success") + } + + // Check {Create, BeginUpdate, EndUpdate, Close} works. + f = nf() + t.Log(f.Name()) + f.BeginUpdate() + if err := f.EndUpdate(); err != nil { + f.Close() + t.Fatal(err) + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } +} + +func TestFilerTruncate(t *testing.T) { + testFilerTruncate(t, newFileFiler) + testFilerTruncate(t, newOSFileFiler) + testFilerTruncate(t, newMemFiler) + testFilerTruncate(t, nwBitFiler) + testFilerTruncate(t, newRollbackFiler) +} + +func testFilerTruncate(t *testing.T, nf newFunc) { + f := nf() + t.Log(f.Name()) + defer func() { + if err := f.Close(); err != nil { + t.Error(err) + } + }() + + if _, ok := f.(*RollbackFiler); ok { + if err := f.BeginUpdate(); err != nil { + t.Fatal(err) + } + + defer func() { + if err := f.EndUpdate(); err != nil { + t.Error(err) + } + }() + } + + // Check Truncate works. + sz := int64(1e6) + if err := f.Truncate(sz); err != nil { + t.Error(err) + return + } + + fsz, err := f.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := fsz, sz; g != e { + t.Error(g, e) + return + } + + sz *= 2 + if err := f.Truncate(sz); err != nil { + t.Error(err) + return + } + + fsz, err = f.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := fsz, sz; g != e { + t.Error(g, e) + return + } + + sz = 0 + if err := f.Truncate(sz); err != nil { + t.Error(err) + return + } + + fsz, err = f.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := fsz, sz; g != e { + t.Error(g, e) + return + } + + // Check Truncate(-1) doesn't work. + sz = -1 + if err := f.Truncate(sz); err == nil { + t.Error(err) + return + } + +} + +func TestFilerReadAtWriteAt(t *testing.T) { + testFilerReadAtWriteAt(t, newFileFiler) + testFilerReadAtWriteAt(t, newOSFileFiler) + testFilerReadAtWriteAt(t, newMemFiler) + testFilerReadAtWriteAt(t, nwBitFiler) + testFilerReadAtWriteAt(t, newRollbackFiler) +} + +func testFilerReadAtWriteAt(t *testing.T, nf newFunc) { + f := nf() + t.Log(f.Name()) + defer func() { + if err := f.Close(); err != nil { + t.Error(err) + } + }() + + if _, ok := f.(*RollbackFiler); ok { + if err := f.BeginUpdate(); err != nil { + t.Fatal(err) + } + + defer func() { + if err := f.EndUpdate(); err != nil { + t.Error(err) + } + }() + } + + const ( + N = 1 << 16 + M = 2e2 + ) + + s := make([]byte, N) + e := make([]byte, N) + rnd := rand.New(rand.NewSource(42)) + for i := range e { + s[i] = byte(rnd.Intn(256)) + } + n2 := 0 + for i := 0; i < M; i++ { + var from, to int + for { + from = rnd.Intn(N) + to = rnd.Intn(N) + if from != to { + break + } + } + if from > to { + from, to = to, from + } + for i := range s[from:to] { + s[from+i] = byte(rnd.Intn(256)) + } + copy(e[from:to], s[from:to]) + if to > n2 { + n2 = to + } + n, err := f.WriteAt(s[from:to], int64(from)) + if err != nil { + t.Error(err) + return + } + + if g, e := n, to-from; g != e { + t.Error(g, e) + return + } + } + + fsz, err := f.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := fsz, int64(n2); g != e { + t.Error(g, e) + return + } + + b := make([]byte, n2) + for i := 0; i <= M; i++ { + from := rnd.Intn(n2) + to := rnd.Intn(n2) + if from > to { + from, to = to, from + } + if i == M { + from, to = 0, n2 + } + n, err := f.ReadAt(b[from:to], int64(from)) + if err != nil && (!fileutil.IsEOF(err) && n != 0) { + fsz, err = f.Size() + if err != nil { + t.Error(err) + return + } + + t.Error(fsz, from, to, err) + return + } + + if g, e := n, to-from; g != e { + t.Error(g, e) + return + } + + if g, e := b[from:to], e[from:to]; !bytes.Equal(g, e) { + if x, ok := f.(*MemFiler); ok { + for i := int64(0); i <= 3; i++ { + t.Logf("pg %d\n----\n%s", i, hex.Dump(x.m[i][:])) + } + } + t.Errorf( + "i %d from %d to %d len(g) %d len(e) %d\n---- got ----\n%s\n---- exp ----\n%s", + i, from, to, len(g), len(e), hex.Dump(g), hex.Dump(e), + ) + return + } + } + + mf, ok := f.(*MemFiler) + if !ok { + return + } + + buf := &bytes.Buffer{} + if _, err := mf.WriteTo(buf); err != nil { + t.Error(err) + return + } + + if g, e := buf.Bytes(), e[:n2]; !bytes.Equal(g, e) { + t.Errorf("\nlen %d\n%s\nlen %d\n%s", len(g), hex.Dump(g), len(e), hex.Dump(e)) + return + } + + if err := mf.Truncate(0); err != nil { + t.Error(err) + return + } + + if _, err := mf.ReadFrom(buf); err != nil { + t.Error(err) + return + } + + roundTrip := make([]byte, n2) + if n, err := mf.ReadAt(roundTrip, 0); err != nil && n == 0 { + t.Error(err) + return + } + + if g, e := roundTrip, e[:n2]; !bytes.Equal(g, e) { + t.Errorf("\nlen %d\n%s\nlen %d\n%s", len(g), hex.Dump(g), len(e), hex.Dump(e)) + return + } +} + +func TestInnerFiler(t *testing.T) { + testInnerFiler(t, newFileFiler) + testInnerFiler(t, newOSFileFiler) + testInnerFiler(t, newMemFiler) + testInnerFiler(t, nwBitFiler) + testInnerFiler(t, newRollbackFiler) +} + +func testInnerFiler(t *testing.T, nf newFunc) { + const ( + HDR_SIZE = 42 + LONG_OFF = 3330 + ) + outer := nf() + t.Log(outer.Name()) + inner := NewInnerFiler(outer, HDR_SIZE) + defer func() { + if err := outer.Close(); err != nil { + t.Error(err) + } + }() + + if _, ok := outer.(*RollbackFiler); ok { + if err := outer.BeginUpdate(); err != nil { + t.Fatal(err) + } + + defer func() { + if err := outer.EndUpdate(); err != nil { + t.Error(err) + } + }() + } + + b := []byte{2, 5, 11} + n, err := inner.WriteAt(b, -1) + if err == nil { + t.Error("unexpected success") + return + } + + n, err = inner.ReadAt(make([]byte, 10), -1) + if err == nil { + t.Error("unexpected success") + return + } + + n, err = inner.WriteAt(b, 0) + if err != nil { + t.Error(err) + return + } + + if g, e := n, len(b); g != e { + t.Error(g, e) + return + } + + osz, err := outer.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := osz, int64(HDR_SIZE+3); g != e { + t.Error(g, e) + return + } + + isz, err := inner.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := isz, int64(3); g != e { + t.Error(g, e) + return + } + + rbuf := make([]byte, 3) + if n, err = outer.ReadAt(rbuf, 0); err != nil && n == 0 { + t.Error(err) + return + } + + if g, e := n, len(rbuf); g != e { + t.Error(g, e) + return + } + + if g, e := rbuf, make([]byte, 3); !bytes.Equal(g, e) { + t.Error(g, e) + } + + rbuf = make([]byte, 3) + if n, err = outer.ReadAt(rbuf, HDR_SIZE); err != nil && n == 0 { + t.Error(err) + return + } + + if g, e := n, len(rbuf); g != e { + t.Error(g, e) + return + } + + if g, e := rbuf, []byte{2, 5, 11}; !bytes.Equal(g, e) { + t.Error(g, e) + } + + rbuf = make([]byte, 3) + if n, err = inner.ReadAt(rbuf, 0); err != nil && n == 0 { + t.Error(err) + return + } + + if g, e := n, len(rbuf); g != e { + t.Error(g, e) + return + } + + if g, e := rbuf, []byte{2, 5, 11}; !bytes.Equal(g, e) { + t.Error(g, e) + } + + b = []byte{22, 55, 111} + if n, err = inner.WriteAt(b, LONG_OFF); err != nil { + t.Error(err) + return + } + + if g, e := n, len(b); g != e { + t.Error(g, e) + return + } + + osz, err = outer.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := osz, int64(HDR_SIZE+LONG_OFF+3); g != e { + t.Error(g, e) + return + } + + isz, err = inner.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := isz, int64(LONG_OFF+3); g != e { + t.Error(g, e) + return + } + + rbuf = make([]byte, 3) + if n, err = outer.ReadAt(rbuf, HDR_SIZE+LONG_OFF); err != nil && n == 0 { + t.Error(err) + return + } + + if g, e := n, len(rbuf); g != e { + t.Error(g, e) + return + } + + if g, e := rbuf, []byte{22, 55, 111}; !bytes.Equal(g, e) { + t.Error(g, e) + } + + rbuf = make([]byte, 3) + if n, err = inner.ReadAt(rbuf, LONG_OFF); err != nil && n == 0 { + t.Error(err) + return + } + + if g, e := n, len(rbuf); g != e { + t.Error(g, e) + return + } + + if g, e := rbuf, []byte{22, 55, 111}; !bytes.Equal(g, e) { + t.Error(g, e) + return + } + + if err = inner.Truncate(1); err != nil { + t.Error(err) + return + } + + isz, err = inner.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := isz, int64(1); g != e { + t.Error(g, e) + return + } + + osz, err = outer.Size() + if err != nil { + t.Error(err) + return + } + + if g, e := osz, int64(HDR_SIZE+1); g != e { + t.Error(g, e) + return + } +} + +func TestFileReadAtHole(t *testing.T) { + testFileReadAtHole(t, newFileFiler) + testFileReadAtHole(t, newOSFileFiler) + testFileReadAtHole(t, newMemFiler) + testFileReadAtHole(t, nwBitFiler) + testFileReadAtHole(t, newRollbackFiler) +} + +func testFileReadAtHole(t *testing.T, nf newFunc) { + f := nf() + t.Log(f.Name()) + defer func() { + if err := f.Close(); err != nil { + t.Error(err) + } + }() + + if _, ok := f.(*RollbackFiler); ok { + if err := f.BeginUpdate(); err != nil { + t.Fatal(err) + } + + defer func() { + if err := f.EndUpdate(); err != nil { + t.Error(err) + } + }() + } + + n, err := f.WriteAt([]byte{1}, 40000) + if err != nil { + t.Error(err) + return + } + + if n != 1 { + t.Error(n) + return + } + + n, err = f.ReadAt(make([]byte, 1000), 20000) + if err != nil { + t.Error(err) + return + } + + if n != 1000 { + t.Error(n) + return + } +} + +func BenchmarkMemFilerWrSeq(b *testing.B) { + b.StopTimer() + buf := make([]byte, filerTestChunkSize) + for i := range buf { + buf[i] = byte(rand.Int()) + } + f := newMemFiler() + runtime.GC() + b.StartTimer() + var ofs int64 + for i := 0; i < b.N; i++ { + _, err := f.WriteAt(buf, ofs) + if err != nil { + b.Fatal(err) + } + + ofs = (ofs + filerTestChunkSize) % filerTotalSize + } +} + +func BenchmarkMemFilerRdSeq(b *testing.B) { + b.StopTimer() + buf := make([]byte, filerTestChunkSize) + for i := range buf { + buf[i] = byte(rand.Int()) + } + f := newMemFiler() + var ofs int64 + for i := 0; i < b.N; i++ { + _, err := f.WriteAt(buf, ofs) + if err != nil { + b.Fatal(err) + } + + ofs = (ofs + filerTestChunkSize) % filerTotalSize + } + runtime.GC() + b.StartTimer() + ofs = 0 + for i := 0; i < b.N; i++ { + n, err := f.ReadAt(buf, ofs) + if err != nil && n == 0 { + b.Fatal(err) + } + + ofs = (ofs + filerTestChunkSize) % filerTotalSize + } +} + +func BenchmarkMemFilerWrRand(b *testing.B) { + b.StopTimer() + rng := rand.New(rand.NewSource(42)) + f := newMemFiler() + var bytes int64 + + var ofs, runs []int + for i := 0; i < b.N; i++ { + ofs = append(ofs, rng.Intn(1<<31-1)) + runs = append(runs, rng.Intn(1<<31-1)%(2*pgSize)) + } + data := make([]byte, 2*pgSize) + for i := range data { + data[i] = byte(rng.Int()) + } + + runtime.GC() + b.StartTimer() + for i, v := range ofs { + n := runs[i] + bytes += int64(n) + f.WriteAt(data[:n], int64(v)) + } + b.StopTimer() +} + +func BenchmarkMemFilerRdRand(b *testing.B) { + b.StopTimer() + rng := rand.New(rand.NewSource(42)) + f := newMemFiler() + var bytes int64 + + var ofs, runs []int + for i := 0; i < b.N; i++ { + ofs = append(ofs, rng.Intn(1<<31-1)) + runs = append(runs, rng.Intn(1<<31-1)%(2*pgSize)) + } + data := make([]byte, 2*pgSize) + for i := range data { + data[i] = byte(rng.Int()) + } + + for i, v := range ofs { + n := runs[i] + bytes += int64(n) + f.WriteAt(data[:n], int64(v)) + } + + runtime.GC() + b.StartTimer() + for _, v := range ofs { + f.ReadAt(data, int64(v)) + } + b.StopTimer() +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/gb.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/gb.go new file mode 100644 index 00000000000..e9090a5473a --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/gb.go @@ -0,0 +1,812 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Utilities to encode/decode and collate Go predeclared scalar types (and the +// typeless nil and []byte). The encoding format is a variation of the one +// used by the "encoding/gob" package. + +package lldb + +import ( + "bytes" + "fmt" + "math" + + "github.com/cznic/mathutil" +) + +const ( + gbNull = iota // 0x00 + gbFalse // 0x01 + gbTrue // 0x02 + gbFloat0 // 0x03 + gbFloat1 // 0x04 + gbFloat2 // 0x05 + gbFloat3 // 0x06 + gbFloat4 // 0x07 + gbFloat5 // 0x08 + gbFloat6 // 0x09 + gbFloat7 // 0x0a + gbFloat8 // 0x0b + gbComplex0 // 0x0c + gbComplex1 // 0x0d + gbComplex2 // 0x0e + gbComplex3 // 0x0f + gbComplex4 // 0x10 + gbComplex5 // 0x11 + gbComplex6 // 0x12 + gbComplex7 // 0x13 + gbComplex8 // 0x14 + gbBytes00 // 0x15 + gbBytes01 // 0x16 + gbBytes02 // 0x17 + gbBytes03 // 0x18 + gbBytes04 // 0x19 + gbBytes05 // 0x1a + gbBytes06 // 0x1b + gbBytes07 // 0x1c + gbBytes08 // 0x1d + gbBytes09 // 0x1e + gbBytes10 // 0x1f + gbBytes11 // 0x20 + gbBytes12 // 0x21 + gbBytes13 // 0x22 + gbBytes14 // 0x23 + gbBytes15 // 0x24 + gbBytes16 // 0x25 + gbBytes17 // Ox26 + gbBytes1 // 0x27 + gbBytes2 // 0x28: Offset by one to allow 64kB sized []byte. + gbString00 // 0x29 + gbString01 // 0x2a + gbString02 // 0x2b + gbString03 // 0x2c + gbString04 // 0x2d + gbString05 // 0x2e + gbString06 // 0x2f + gbString07 // 0x30 + gbString08 // 0x31 + gbString09 // 0x32 + gbString10 // 0x33 + gbString11 // 0x34 + gbString12 // 0x35 + gbString13 // 0x36 + gbString14 // 0x37 + gbString15 // 0x38 + gbString16 // 0x39 + gbString17 // 0x3a + gbString1 // 0x3b + gbString2 // 0x3c + gbUintP1 // 0x3d + gbUintP2 // 0x3e + gbUintP3 // 0x3f + gbUintP4 // 0x40 + gbUintP5 // 0x41 + gbUintP6 // 0x42 + gbUintP7 // 0x43 + gbUintP8 // 0x44 + gbIntM8 // 0x45 + gbIntM7 // 0x46 + gbIntM6 // 0x47 + gbIntM5 // 0x48 + gbIntM4 // 0x49 + gbIntM3 // 0x4a + gbIntM2 // 0x4b + gbIntM1 // 0x4c + gbIntP1 // 0x4d + gbIntP2 // 0x4e + gbIntP3 // 0x4f + gbIntP4 // 0x50 + gbIntP5 // 0x51 + gbIntP6 // 0x52 + gbIntP7 // 0x53 + gbIntP8 // 0x54 + gbInt0 // 0x55 + + gbIntMax = 255 - gbInt0 // 0xff == 170 +) + +// EncodeScalars encodes a vector of predeclared scalar type values to a +// []byte, making it suitable to store it as a "record" in a DB or to use it as +// a key of a BTree. +func EncodeScalars(scalars ...interface{}) (b []byte, err error) { + for _, scalar := range scalars { + switch x := scalar.(type) { + default: + return nil, &ErrINVAL{"EncodeScalars: unsupported type", fmt.Sprintf("%T in `%#v`", x, scalars)} + + case nil: + b = append(b, gbNull) + + case bool: + switch x { + case false: + b = append(b, gbFalse) + case true: + b = append(b, gbTrue) + } + + case float32: + encFloat(float64(x), &b) + case float64: + encFloat(x, &b) + + case complex64: + encComplex(complex128(x), &b) + case complex128: + encComplex(x, &b) + + case string: + n := len(x) + if n <= 17 { + b = append(b, byte(gbString00+n)) + b = append(b, []byte(x)...) + break + } + + if n > 65535 { + return nil, fmt.Errorf("EncodeScalars: cannot encode string of length %d (limit 65536)", n) + } + + pref := byte(gbString1) + if n > 255 { + pref++ + } + b = append(b, pref) + encUint0(uint64(n), &b) + b = append(b, []byte(x)...) + + case int8: + encInt(int64(x), &b) + case int16: + encInt(int64(x), &b) + case int32: + encInt(int64(x), &b) + case int64: + encInt(x, &b) + case int: + encInt(int64(x), &b) + + case uint8: + encUint(uint64(x), &b) + case uint16: + encUint(uint64(x), &b) + case uint32: + encUint(uint64(x), &b) + case uint64: + encUint(x, &b) + case uint: + encUint(uint64(x), &b) + case []byte: + n := len(x) + if n <= 17 { + b = append(b, byte(gbBytes00+n)) + b = append(b, []byte(x)...) + break + } + + if n > 655356 { + return nil, fmt.Errorf("EncodeScalars: cannot encode []byte of length %d (limit 65536)", n) + } + + pref := byte(gbBytes1) + if n > 255 { + pref++ + } + b = append(b, pref) + if n <= 255 { + b = append(b, byte(n)) + } else { + n-- + b = append(b, byte(n>>8), byte(n)) + } + b = append(b, x...) + } + } + return +} + +func encComplex(f complex128, b *[]byte) { + encFloatPrefix(gbComplex0, real(f), b) + encFloatPrefix(gbComplex0, imag(f), b) +} + +func encFloatPrefix(prefix byte, f float64, b *[]byte) { + u := math.Float64bits(f) + var n uint64 + for i := 0; i < 8; i++ { + n <<= 8 + n |= u & 0xFF + u >>= 8 + } + bits := mathutil.BitLenUint64(n) + if bits == 0 { + *b = append(*b, prefix) + return + } + + // 0 1 2 3 4 5 6 7 8 9 + // . 1 1 1 1 1 1 1 1 2 + encUintPrefix(prefix+1+byte((bits-1)>>3), n, b) +} + +func encFloat(f float64, b *[]byte) { + encFloatPrefix(gbFloat0, f, b) +} + +func encUint0(n uint64, b *[]byte) { + switch { + case n <= 0xff: + *b = append(*b, byte(n)) + case n <= 0xffff: + *b = append(*b, byte(n>>8), byte(n)) + case n <= 0xffffff: + *b = append(*b, byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffff: + *b = append(*b, byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffffff: + *b = append(*b, byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffffffff: + *b = append(*b, byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffffffffff: + *b = append(*b, byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= math.MaxUint64: + *b = append(*b, byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + } +} + +func encUintPrefix(prefix byte, n uint64, b *[]byte) { + *b = append(*b, prefix) + encUint0(n, b) +} + +func encUint(n uint64, b *[]byte) { + bits := mathutil.Max(1, mathutil.BitLenUint64(n)) + encUintPrefix(gbUintP1+byte((bits-1)>>3), n, b) +} + +func encInt(n int64, b *[]byte) { + switch { + case n < -0x100000000000000: + *b = append(*b, byte(gbIntM8), byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n < -0x1000000000000: + *b = append(*b, byte(gbIntM7), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n < -0x10000000000: + *b = append(*b, byte(gbIntM6), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n < -0x100000000: + *b = append(*b, byte(gbIntM5), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n < -0x1000000: + *b = append(*b, byte(gbIntM4), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n < -0x10000: + *b = append(*b, byte(gbIntM3), byte(n>>16), byte(n>>8), byte(n)) + case n < -0x100: + *b = append(*b, byte(gbIntM2), byte(n>>8), byte(n)) + case n < 0: + *b = append(*b, byte(gbIntM1), byte(n)) + case n <= gbIntMax: + *b = append(*b, byte(gbInt0+n)) + case n <= 0xff: + *b = append(*b, gbIntP1, byte(n)) + case n <= 0xffff: + *b = append(*b, gbIntP2, byte(n>>8), byte(n)) + case n <= 0xffffff: + *b = append(*b, gbIntP3, byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffff: + *b = append(*b, gbIntP4, byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffffff: + *b = append(*b, gbIntP5, byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffffffff: + *b = append(*b, gbIntP6, byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= 0xffffffffffffff: + *b = append(*b, gbIntP7, byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + case n <= 0x7fffffffffffffff: + *b = append(*b, gbIntP8, byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + } +} + +func decodeFloat(b []byte) float64 { + var u uint64 + for i, v := range b { + u |= uint64(v) << uint((i+8-len(b))*8) + } + return math.Float64frombits(u) +} + +// DecodeScalars decodes a []byte produced by EncodeScalars. +func DecodeScalars(b []byte) (scalars []interface{}, err error) { + b0 := b + for len(b) != 0 { + switch tag := b[0]; tag { + //default: + //return nil, fmt.Errorf("tag %d(%#x) not supported", b[0], b[0]) + case gbNull: + scalars = append(scalars, nil) + b = b[1:] + case gbFalse: + scalars = append(scalars, false) + b = b[1:] + case gbTrue: + scalars = append(scalars, true) + b = b[1:] + case gbFloat0: + scalars = append(scalars, 0.0) + b = b[1:] + case gbFloat1, gbFloat2, gbFloat3, gbFloat4, gbFloat5, gbFloat6, gbFloat7, gbFloat8: + n := 1 + int(tag) - gbFloat0 + if len(b) < n-1 { + goto corrupted + } + + scalars = append(scalars, decodeFloat(b[1:n])) + b = b[n:] + case gbComplex0, gbComplex1, gbComplex2, gbComplex3, gbComplex4, gbComplex5, gbComplex6, gbComplex7, gbComplex8: + n := 1 + int(tag) - gbComplex0 + if len(b) < n-1 { + goto corrupted + } + + re := decodeFloat(b[1:n]) + b = b[n:] + + if len(b) == 0 { + goto corrupted + } + + tag = b[0] + if tag < gbComplex0 || tag > gbComplex8 { + goto corrupted + } + + n = 1 + int(tag) - gbComplex0 + if len(b) < n-1 { + goto corrupted + } + + scalars = append(scalars, complex(re, decodeFloat(b[1:n]))) + b = b[n:] + case gbBytes00, gbBytes01, gbBytes02, gbBytes03, gbBytes04, + gbBytes05, gbBytes06, gbBytes07, gbBytes08, gbBytes09, + gbBytes10, gbBytes11, gbBytes12, gbBytes13, gbBytes14, + gbBytes15, gbBytes16, gbBytes17: + n := int(tag - gbBytes00) + if len(b) < n+1 { + goto corrupted + } + + scalars = append(scalars, append([]byte(nil), b[1:n+1]...)) + b = b[n+1:] + case gbBytes1: + if len(b) < 2 { + goto corrupted + } + + n := int(b[1]) + b = b[2:] + if len(b) < n { + goto corrupted + } + + scalars = append(scalars, append([]byte(nil), b[:n]...)) + b = b[n:] + case gbBytes2: + if len(b) < 3 { + goto corrupted + } + + n := int(b[1])<<8 | int(b[2]) + 1 + b = b[3:] + if len(b) < n { + goto corrupted + } + + scalars = append(scalars, append([]byte(nil), b[:n]...)) + b = b[n:] + case gbString00, gbString01, gbString02, gbString03, gbString04, + gbString05, gbString06, gbString07, gbString08, gbString09, + gbString10, gbString11, gbString12, gbString13, gbString14, + gbString15, gbString16, gbString17: + n := int(tag - gbString00) + if len(b) < n+1 { + goto corrupted + } + + scalars = append(scalars, string(b[1:n+1])) + b = b[n+1:] + case gbString1: + if len(b) < 2 { + goto corrupted + } + + n := int(b[1]) + b = b[2:] + if len(b) < n { + goto corrupted + } + + scalars = append(scalars, string(b[:n])) + b = b[n:] + case gbString2: + if len(b) < 3 { + goto corrupted + } + + n := int(b[1])<<8 | int(b[2]) + b = b[3:] + if len(b) < n { + goto corrupted + } + + scalars = append(scalars, string(b[:n])) + b = b[n:] + case gbUintP1, gbUintP2, gbUintP3, gbUintP4, gbUintP5, gbUintP6, gbUintP7, gbUintP8: + b = b[1:] + n := 1 + int(tag) - gbUintP1 + if len(b) < n { + goto corrupted + } + + var u uint64 + for _, v := range b[:n] { + u = u<<8 | uint64(v) + } + scalars = append(scalars, u) + b = b[n:] + case gbIntM8, gbIntM7, gbIntM6, gbIntM5, gbIntM4, gbIntM3, gbIntM2, gbIntM1: + b = b[1:] + n := 8 - (int(tag) - gbIntM8) + if len(b) < n { + goto corrupted + } + u := uint64(math.MaxUint64) + for _, v := range b[:n] { + u = u<<8 | uint64(v) + } + scalars = append(scalars, int64(u)) + b = b[n:] + case gbIntP1, gbIntP2, gbIntP3, gbIntP4, gbIntP5, gbIntP6, gbIntP7, gbIntP8: + b = b[1:] + n := 1 + int(tag) - gbIntP1 + if len(b) < n { + goto corrupted + } + + i := int64(0) + for _, v := range b[:n] { + i = i<<8 | int64(v) + } + scalars = append(scalars, i) + b = b[n:] + default: + scalars = append(scalars, int64(b[0])-gbInt0) + b = b[1:] + } + } + return append([]interface{}(nil), scalars...), nil + +corrupted: + return nil, &ErrDecodeScalars{append([]byte(nil), b0...), len(b0) - len(b)} +} + +func collateComplex(x, y complex128) int { + switch rx, ry := real(x), real(y); { + case rx < ry: + return -1 + case rx == ry: + switch ix, iy := imag(x), imag(y); { + case ix < iy: + return -1 + case ix == iy: + return 0 + case ix > iy: + return 1 + } + } + //case rx > ry: + return 1 +} + +func collateFloat(x, y float64) int { + switch { + case x < y: + return -1 + case x == y: + return 0 + } + //case x > y: + return 1 +} + +func collateInt(x, y int64) int { + switch { + case x < y: + return -1 + case x == y: + return 0 + } + //case x > y: + return 1 +} + +func collateUint(x, y uint64) int { + switch { + case x < y: + return -1 + case x == y: + return 0 + } + //case x > y: + return 1 +} + +func collateIntUint(x int64, y uint64) int { + if y > math.MaxInt64 { + return -1 + } + + return collateInt(x, int64(y)) +} + +func collateUintInt(x uint64, y int64) int { + return -collateIntUint(y, x) +} + +func collateType(i interface{}) (r interface{}, err error) { + switch x := i.(type) { + default: + return nil, fmt.Errorf("invalid collate type %T", x) + case nil: + return i, nil + case bool: + return i, nil + case int8: + return int64(x), nil + case int16: + return int64(x), nil + case int32: + return int64(x), nil + case int64: + return i, nil + case int: + return int64(x), nil + case uint8: + return uint64(x), nil + case uint16: + return uint64(x), nil + case uint32: + return uint64(x), nil + case uint64: + return i, nil + case uint: + return uint64(x), nil + case float32: + return float64(x), nil + case float64: + return i, nil + case complex64: + return complex128(x), nil + case complex128: + return i, nil + case []byte: + return i, nil + case string: + return i, nil + } +} + +// Collate collates two arrays of Go predeclared scalar types (and the typeless +// nil or []byte). If any other type appears in x or y, Collate will return a +// non nil error. String items are collated using strCollate or lexically +// byte-wise (as when using Go comparison operators) when strCollate is nil. +// []byte items are collated using bytes.Compare. +// +// Collate returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +// The same value as defined above must be returned from strCollate. +// +// The "outer" ordering is: nil, bool, number, []byte, string. IOW, nil is +// "smaller" than anything else except other nil, numbers collate before +// []byte, []byte collate before strings, etc. +// +// Integers and real numbers collate as expected in math. However, complex +// numbers are not ordered in Go. Here the ordering is defined: Complex numbers +// are in comparison considered first only by their real part. Iff the result +// is equality then the imaginary part is used to determine the ordering. In +// this "second order" comparing, integers and real numbers are considered as +// complex numbers with a zero imaginary part. +func Collate(x, y []interface{}, strCollate func(string, string) int) (r int, err error) { + nx, ny := len(x), len(y) + + switch { + case nx == 0 && ny != 0: + return -1, nil + case nx == 0 && ny == 0: + return 0, nil + case nx != 0 && ny == 0: + return 1, nil + } + + r = 1 + if nx > ny { + x, y, r = y, x, -r + } + + var c int + for i, xi0 := range x { + yi0 := y[i] + xi, err := collateType(xi0) + if err != nil { + return 0, err + } + + yi, err := collateType(yi0) + if err != nil { + return 0, err + } + + switch x := xi.(type) { + default: + panic(fmt.Errorf("internal error: %T", x)) + + case nil: + switch yi.(type) { + case nil: + // nop + default: + return -r, nil + } + + case bool: + switch y := yi.(type) { + case nil: + return r, nil + case bool: + switch { + case !x && y: + return -r, nil + case x == y: + // nop + case x && !y: + return r, nil + } + default: + return -r, nil + } + + case int64: + switch y := yi.(type) { + case nil, bool: + return r, nil + case int64: + c = collateInt(x, y) + case uint64: + c = collateIntUint(x, y) + case float64: + c = collateFloat(float64(x), y) + case complex128: + c = collateComplex(complex(float64(x), 0), y) + case []byte: + return -r, nil + case string: + return -r, nil + } + + if c != 0 { + return c * r, nil + } + + case uint64: + switch y := yi.(type) { + case nil, bool: + return r, nil + case int64: + c = collateUintInt(x, y) + case uint64: + c = collateUint(x, y) + case float64: + c = collateFloat(float64(x), y) + case complex128: + c = collateComplex(complex(float64(x), 0), y) + case []byte: + return -r, nil + case string: + return -r, nil + } + + if c != 0 { + return c * r, nil + } + + case float64: + switch y := yi.(type) { + case nil, bool: + return r, nil + case int64: + c = collateFloat(x, float64(y)) + case uint64: + c = collateFloat(x, float64(y)) + case float64: + c = collateFloat(x, y) + case complex128: + c = collateComplex(complex(x, 0), y) + case []byte: + return -r, nil + case string: + return -r, nil + } + + if c != 0 { + return c * r, nil + } + + case complex128: + switch y := yi.(type) { + case nil, bool: + return r, nil + case int64: + c = collateComplex(x, complex(float64(y), 0)) + case uint64: + c = collateComplex(x, complex(float64(y), 0)) + case float64: + c = collateComplex(x, complex(y, 0)) + case complex128: + c = collateComplex(x, y) + case []byte: + return -r, nil + case string: + return -r, nil + } + + if c != 0 { + return c * r, nil + } + + case []byte: + switch y := yi.(type) { + case nil, bool, int64, uint64, float64, complex128: + return r, nil + case []byte: + c = bytes.Compare(x, y) + case string: + return -r, nil + } + + if c != 0 { + return c * r, nil + } + + case string: + switch y := yi.(type) { + case nil, bool, int64, uint64, float64, complex128: + return r, nil + case []byte: + return r, nil + case string: + switch { + case strCollate != nil: + c = strCollate(x, y) + case x < y: + return -r, nil + case x == y: + c = 0 + case x > y: + return r, nil + } + } + + if c != 0 { + return c * r, nil + } + } + } + + if nx == ny { + return 0, nil + } + + return -r, nil +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/gb_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/gb_test.go new file mode 100644 index 00000000000..a9923ca56b8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/gb_test.go @@ -0,0 +1,364 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Utilities to encode/decode and collate Go predeclared scalar types. The +// encoding format reused the one used by the "encoding/gob" package. + +package lldb + +import ( + "bytes" + "math" + "testing" +) + +const s256 = "" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + + "0123456789abcdef" + +func TestEncodeDecodeScalars(t *testing.T) { + table := []struct{ v, exp interface{} }{ + {nil, "00"}, + {false, "01"}, + {true, "02"}, + {math.Float64frombits(0), []byte{gbFloat0}}, + {17., []byte{gbFloat2, 0x31, 0x40}}, + {math.Float64frombits(0x4031320000000000), []byte{gbFloat3, 0x32, 0x31, 0x40}}, + {math.Float64frombits(0x4031323300000000), []byte{gbFloat4, 0x33, 0x32, 0x31, 0x40}}, + {math.Float64frombits(0x4031323334000000), []byte{gbFloat5, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {math.Float64frombits(0x4031323334350000), []byte{gbFloat6, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {math.Float64frombits(0x4031323334353600), []byte{gbFloat7, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {math.Float64frombits(0x4031323334353637), []byte{gbFloat8, 0x37, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {0 + 0i, []byte{gbComplex0, gbComplex0}}, + {17 + 17i, []byte{gbComplex2, 0x31, 0x40, gbComplex2, 0x31, 0x40}}, + {complex(math.Float64frombits(0x4041420000000000), math.Float64frombits(0x4031320000000000)), []byte{gbComplex3, 0x42, 0x41, 0x40, gbComplex3, 0x32, 0x31, 0x40}}, + {complex(math.Float64frombits(0x4041424300000000), math.Float64frombits(0x4031323300000000)), []byte{gbComplex4, 0x43, 0x42, 0x41, 0x40, gbComplex4, 0x33, 0x32, 0x31, 0x40}}, + {complex(math.Float64frombits(0x4041424344000000), math.Float64frombits(0x4031323334000000)), []byte{gbComplex5, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex5, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {complex(math.Float64frombits(0x4041424344450000), math.Float64frombits(0x4031323334350000)), []byte{gbComplex6, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex6, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {complex(math.Float64frombits(0x4041424344454600), math.Float64frombits(0x4031323334353600)), []byte{gbComplex7, 0x46, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex7, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {complex(math.Float64frombits(0x4041424344454647), math.Float64frombits(0x4031323334353637)), []byte{gbComplex8, 0x47, 0x46, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex8, 0x37, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}}, + {[]byte(""), []byte{gbBytes00}}, + {[]byte("f"), []byte{gbBytes01, 'f'}}, + {[]byte("fo"), []byte{gbBytes02, 'f', 'o'}}, + {[]byte("0123456789abcdefx"), []byte{gbBytes17, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x'}}, + {[]byte("0123456789abcdefxy"), []byte{gbBytes1, 18, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x', 'y'}}, + {[]byte(s256[:255]), append([]byte{gbBytes1, 0xff}, []byte(s256[:255])...)}, + {[]byte(s256), append([]byte{gbBytes2, 0x00, 0xff}, []byte(s256)...)}, + {"", []byte{gbString00}}, + {"f", []byte{gbString01, 'f'}}, + {"fo", []byte{gbString02, 'f', 'o'}}, + {"0123456789abcdefx", []byte{gbString17, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x'}}, + {"0123456789abcdefxy", []byte{gbString1, 18, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x', 'y'}}, + {s256[:255], append([]byte{gbString1, 0xff}, []byte(s256[:255])...)}, + {s256, append([]byte{gbString2, 0x01, 0x00}, []byte(s256)...)}, + {uint64(0xff), []byte{gbUintP1, 255}}, + {uint64(0xffff), []byte{gbUintP2, 255, 255}}, + {uint64(0xffffff), []byte{gbUintP3, 255, 255, 255}}, + {uint64(0xffffffff), []byte{gbUintP4, 255, 255, 255, 255}}, + {uint64(0xffffffffff), []byte{gbUintP5, 255, 255, 255, 255, 255}}, + {uint64(0xffffffffffff), []byte{gbUintP6, 255, 255, 255, 255, 255, 255}}, + {uint64(0xffffffffffffff), []byte{gbUintP7, 255, 255, 255, 255, 255, 255, 255}}, + {uint64(0xffffffffffffffff), []byte{gbUintP8, 255, 255, 255, 255, 255, 255, 255, 255}}, + {int64(math.MinInt64), []byte{gbIntM8, 128, 0, 0, 0, 0, 0, 0, 0}}, + {-int64(0x100000000000000), []byte{gbIntM7, 0, 0, 0, 0, 0, 0, 0}}, + {-int64(0x1000000000000), []byte{gbIntM6, 0, 0, 0, 0, 0, 0}}, + {-int64(0x10000000000), []byte{gbIntM5, 0, 0, 0, 0, 0}}, + {-int64(0x100000000), []byte{gbIntM4, 0, 0, 0, 0}}, + {-int64(0x1000000), []byte{gbIntM3, 0, 0, 0}}, + {-int64(0x10000), []byte{gbIntM2, 0, 0}}, + {-int64(0x100), []byte{gbIntM1, 0}}, + {-int64(0xff), []byte{gbIntM1, 1}}, + {-int64(1), []byte{gbIntM1, 255}}, + {int64(gbIntMax + 1), []byte{gbIntP1, gbIntMax + 1}}, + {int64(0xff), []byte{gbIntP1, 255}}, + {int64(0xffff), []byte{gbIntP2, 255, 255}}, + {int64(0xffffff), []byte{gbIntP3, 255, 255, 255}}, + {int64(0xffffffff), []byte{gbIntP4, 255, 255, 255, 255}}, + {int64(0xffffffffff), []byte{gbIntP5, 255, 255, 255, 255, 255}}, + {int64(0xffffffffffff), []byte{gbIntP6, 255, 255, 255, 255, 255, 255}}, + {int64(0xffffffffffffff), []byte{gbIntP7, 255, 255, 255, 255, 255, 255, 255}}, + {int64(0x7fffffffffffffff), []byte{gbIntP8, 127, 255, 255, 255, 255, 255, 255, 255}}, + {int64(0), []byte{0 + gbInt0}}, + {int64(1), []byte{1 + gbInt0}}, + {int64(2), []byte{2 + gbInt0}}, + {int64(gbIntMax - 2), "fd"}, + {int64(gbIntMax - 1), "fe"}, + {int64(gbIntMax), "ff"}, + } + + for i, v := range table { + g, err := EncodeScalars(v.v) + if err != nil { + t.Fatal(i, err) + } + + var e []byte + switch x := v.exp.(type) { + case string: + e = s2b(x) + case []byte: + e = x + } + + if !bytes.Equal(g, e) { + t.Fatalf("%d %v\n|% 02x|\n|% 02x|", i, v.v, g, e) + } + + t.Logf("%#v |% 02x|", v.v, g) + + dec, err := DecodeScalars(g) + if err != nil { + t.Fatal(err) + } + + if g, e := len(dec), 1; g != e { + t.Fatalf("%d %d %#v", g, e, dec) + } + + if g, ok := dec[0].([]byte); ok { + if e := v.v.([]byte); !bytes.Equal(g, e) { + t.Fatal(g, e) + } + + continue + } + + if g, e := dec[0], v.v; g != e { + t.Fatal(g, e) + } + } +} + +func strcmp(a, b string) (r int) { + if a < b { + return -1 + } + + if a == b { + return 0 + } + + return 1 +} + +func TestCollateScalars(t *testing.T) { + // all cases must return -1 + table := []struct{ x, y []interface{} }{ + {[]interface{}{}, []interface{}{1}}, + {[]interface{}{1}, []interface{}{2}}, + {[]interface{}{1, 2}, []interface{}{2, 3}}, + + {[]interface{}{nil}, []interface{}{nil, true}}, + {[]interface{}{nil}, []interface{}{false}}, + {[]interface{}{nil}, []interface{}{nil, 1}}, + {[]interface{}{nil}, []interface{}{1}}, + {[]interface{}{nil}, []interface{}{nil, uint(1)}}, + {[]interface{}{nil}, []interface{}{uint(1)}}, + {[]interface{}{nil}, []interface{}{nil, 3.14}}, + {[]interface{}{nil}, []interface{}{3.14}}, + {[]interface{}{nil}, []interface{}{nil, 3.14 + 1i}}, + {[]interface{}{nil}, []interface{}{3.14 + 1i}}, + {[]interface{}{nil}, []interface{}{nil, []byte("foo")}}, + {[]interface{}{nil}, []interface{}{[]byte("foo")}}, + {[]interface{}{nil}, []interface{}{nil, "foo"}}, + {[]interface{}{nil}, []interface{}{"foo"}}, + + {[]interface{}{false}, []interface{}{false, false}}, + {[]interface{}{false}, []interface{}{false, true}}, + {[]interface{}{false}, []interface{}{true}}, + {[]interface{}{false}, []interface{}{false, 1}}, + {[]interface{}{false}, []interface{}{1}}, + {[]interface{}{false}, []interface{}{false, uint(1)}}, + {[]interface{}{false}, []interface{}{uint(1)}}, + {[]interface{}{false}, []interface{}{false, 1.5}}, + {[]interface{}{false}, []interface{}{1.5}}, + {[]interface{}{false}, []interface{}{false, 1.5 + 3i}}, + {[]interface{}{false}, []interface{}{1.5 + 3i}}, + {[]interface{}{false}, []interface{}{false, []byte("foo")}}, + {[]interface{}{false}, []interface{}{[]byte("foo")}}, + {[]interface{}{false}, []interface{}{false, "foo"}}, + {[]interface{}{false}, []interface{}{"foo"}}, + + {[]interface{}{1}, []interface{}{1, 2}}, + {[]interface{}{1}, []interface{}{1, 1}}, + {[]interface{}{1}, []interface{}{1, uint(2)}}, + {[]interface{}{1}, []interface{}{uint(2)}}, + {[]interface{}{1}, []interface{}{1, 1.1}}, + {[]interface{}{1}, []interface{}{1.1}}, + {[]interface{}{1}, []interface{}{1, 1.1 + 2i}}, + {[]interface{}{1}, []interface{}{1.1 + 2i}}, + {[]interface{}{1}, []interface{}{1, []byte("foo")}}, + {[]interface{}{1}, []interface{}{[]byte("foo")}}, + {[]interface{}{1}, []interface{}{1, "foo"}}, + {[]interface{}{1}, []interface{}{"foo"}}, + + {[]interface{}{uint(1)}, []interface{}{uint(1), uint(1)}}, + {[]interface{}{uint(1)}, []interface{}{uint(2)}}, + {[]interface{}{uint(1)}, []interface{}{uint(1), 2.}}, + {[]interface{}{uint(1)}, []interface{}{2.}}, + {[]interface{}{uint(1)}, []interface{}{uint(1), 2. + 0i}}, + {[]interface{}{uint(1)}, []interface{}{2. + 0i}}, + {[]interface{}{uint(1)}, []interface{}{uint(1), []byte("foo")}}, + {[]interface{}{uint(1)}, []interface{}{[]byte("foo")}}, + {[]interface{}{uint(1)}, []interface{}{uint(1), "foo"}}, + {[]interface{}{uint(1)}, []interface{}{"foo"}}, + + {[]interface{}{1.}, []interface{}{1., 1}}, + {[]interface{}{1.}, []interface{}{2}}, + {[]interface{}{1.}, []interface{}{1., uint(1)}}, + {[]interface{}{1.}, []interface{}{uint(2)}}, + {[]interface{}{1.}, []interface{}{1., 1.}}, + {[]interface{}{1.}, []interface{}{1.1}}, + {[]interface{}{1.}, []interface{}{1., []byte("foo")}}, + {[]interface{}{1.}, []interface{}{[]byte("foo")}}, + {[]interface{}{1.}, []interface{}{1., "foo"}}, + {[]interface{}{1.}, []interface{}{"foo"}}, + + {[]interface{}{1 + 2i}, []interface{}{1 + 2i, 1}}, + {[]interface{}{1 + 2i}, []interface{}{2}}, + {[]interface{}{1 + 2i}, []interface{}{1 + 2i, uint(1)}}, + {[]interface{}{1 + 2i}, []interface{}{uint(2)}}, + {[]interface{}{1 + 2i}, []interface{}{1 + 2i, 1.1}}, + {[]interface{}{1 + 2i}, []interface{}{1.1}}, + {[]interface{}{1 + 2i}, []interface{}{1 + 2i, []byte("foo")}}, + {[]interface{}{1 + 2i}, []interface{}{[]byte("foo")}}, + {[]interface{}{1 + 2i}, []interface{}{1 + 2i, "foo"}}, + {[]interface{}{1 + 2i}, []interface{}{"foo"}}, + + {[]interface{}{[]byte("bar")}, []interface{}{[]byte("bar"), []byte("bar")}}, + {[]interface{}{[]byte("bar")}, []interface{}{[]byte("foo")}}, + {[]interface{}{[]byte("bar")}, []interface{}{[]byte("c")}}, + {[]interface{}{[]byte("bar")}, []interface{}{[]byte("bas")}}, + {[]interface{}{[]byte("bar")}, []interface{}{[]byte("bara")}}, + + {[]interface{}{[]byte("bar")}, []interface{}{"bap"}}, + {[]interface{}{[]byte("bar")}, []interface{}{"bar"}}, + {[]interface{}{[]byte("bar")}, []interface{}{"bas"}}, + + {[]interface{}{"bar"}, []interface{}{"bar", "bar"}}, + {[]interface{}{"bar"}, []interface{}{"foo"}}, + {[]interface{}{"bar"}, []interface{}{"c"}}, + {[]interface{}{"bar"}, []interface{}{"bas"}}, + {[]interface{}{"bar"}, []interface{}{"bara"}}, + + {[]interface{}{1 + 2i}, []interface{}{1 + 3i}}, + {[]interface{}{int64(math.MaxInt64)}, []interface{}{uint64(math.MaxInt64 + 1)}}, + {[]interface{}{int8(1)}, []interface{}{int16(2)}}, + {[]interface{}{int32(1)}, []interface{}{uint8(2)}}, + {[]interface{}{uint16(1)}, []interface{}{uint32(2)}}, + {[]interface{}{float32(1)}, []interface{}{complex(float32(2), 0)}}, + + // resolved bugs + {[]interface{}{"Customer"}, []interface{}{"Date"}}, + {[]interface{}{"Customer"}, []interface{}{"Items", 1, "Quantity"}}, + } + + more := []interface{}{42, nil, 1, uint(2), 3.0, 4 + 5i, "..."} + + collate := func(x, y []interface{}, strCollate func(string, string) int) (r int) { + var err error + r, err = Collate(x, y, strCollate) + if err != nil { + t.Fatal(err) + } + + return + } + + for _, scf := range []func(string, string) int{nil, strcmp} { + for _, prefix := range more { + for i, test := range table { + var x, y []interface{} + if prefix != 42 { + x = append(x, prefix) + y = append(y, prefix) + } + x = append(x, test.x...) + y = append(y, test.y...) + + // cmp(x, y) == -1 + if g, e := collate(x, y, scf), -1; g != e { + t.Fatal(i, g, e, x, y) + } + + // cmp(y, x) == 1 + if g, e := collate(y, x, scf), 1; g != e { + t.Fatal(i, g, e, y, x) + } + + src := x + for ix := len(src) - 1; ix > 0; ix-- { + if g, e := collate(src[:ix], src[:ix], scf), 0; g != e { + t.Fatal(ix, g, e) + } + + if g, e := collate(src[:ix], src, scf), -1; g != e { + t.Fatal(ix, g, e) + } + + } + + src = y + for ix := len(src) - 1; ix > 0; ix-- { + if g, e := collate(src[:ix], src[:ix], scf), 0; g != e { + t.Fatal(ix, g, e) + } + + if g, e := collate(src[:ix], src, scf), -1; g != e { + t.Fatal(ix, g, e) + } + + } + } + } + } +} + +func TestEncodingBug(t *testing.T) { + bits := uint64(0) + for i := 0; i <= 64; i++ { + encoded, err := EncodeScalars(math.Float64frombits(bits)) + if err != nil { + t.Fatal(err) + } + + t.Logf("bits %016x, enc |% x|", bits, encoded) + decoded, err := DecodeScalars(encoded) + if err != nil { + t.Fatal(err) + } + + if g, e := len(decoded), 1; g != e { + t.Fatal(g, e) + } + + f, ok := decoded[0].(float64) + if !ok { + t.Fatal(err) + } + + if g, e := math.Float64bits(f), bits; g != e { + t.Fatal(err) + } + + t.Log(f) + + bits >>= 1 + bits |= 1 << 63 + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb.go new file mode 100644 index 00000000000..8f77ec8add7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb.go @@ -0,0 +1,155 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lldb (WIP) implements a low level database engine. The database +// model used could be considered a specific implementation of some small(est) +// intersection of models listed in [1]. As a settled term is lacking, it'll be +// called here a 'Virtual memory model' (VMM). +// +// Experimental release notes +// +// This is an experimental release. Don't open a DB from two applications or +// two instances of an application - it will get corrupted (no file locking is +// implemented and this task is delegated to lldb's clients). +// +// WARNING: THE LLDB API IS SUBJECT TO CHANGE. +// +// Filers +// +// A Filer is an abstraction of storage. A Filer may be a part of some process' +// virtual address space, an OS file, a networked, remote file etc. Persistence +// of the storage is optional, opaque to VMM and it is specific to a concrete +// Filer implementation. +// +// Space management +// +// Mechanism to allocate, reallocate (resize), deallocate (and later reclaim +// the unused) contiguous parts of a Filer, called blocks. Blocks are +// identified and referred to by a handle, an int64. +// +// BTrees +// +// In addition to the VMM like services, lldb provides volatile and +// non-volatile BTrees. Keys and values of a BTree are limited in size to 64kB +// each (a bit more actually). Support for larger keys/values, if desired, can +// be built atop a BTree to certain limits. +// +// Handles vs pointers +// +// A handle is the abstracted storage counterpart of a memory address. There +// is one fundamental difference, though. Resizing a block never results in a +// change to the handle which refers to the resized block, so a handle is more +// akin to an unique numeric id/key. Yet it shares one property of pointers - +// handles can be associated again with blocks after the original handle block +// was deallocated. In other words, a handle uniqueness domain is the state of +// the database and is not something comparable to e.g. an ever growing +// numbering sequence. +// +// Also, as with memory pointers, dangling handles can be created and blocks +// overwritten when such handles are used. Using a zero handle to refer to a +// block will not panic; however, the resulting error is effectively the same +// exceptional situation as dereferencing a nil pointer. +// +// Blocks +// +// Allocated/used blocks, are limited in size to only a little bit more than +// 64kB. Bigger semantic entities/structures must be built in lldb's client +// code. The content of a block has no semantics attached, it's only a fully +// opaque `[]byte`. +// +// Scalars +// +// Use of "scalars" applies to EncodeScalars, DecodeScalars and Collate. Those +// first two "to bytes" and "from bytes" functions are suggested for handling +// multi-valued Allocator content items and/or keys/values of BTrees (using +// Collate for keys). Types called "scalar" are: +// +// nil (the typeless one) +// bool +// all integral types: [u]int8, [u]int16, [u]int32, [u]int, [u]int64 +// all floating point types: float32, float64 +// all complex types: complex64, complex128 +// []byte (64kB max) +// string (64kb max) +// +// Specific implementations +// +// Included are concrete implementations of some of the VMM interfaces included +// to ease serving simple client code or for testing and possibly as an +// example. More details in the documentation of such implementations. +// +// [1]: http://en.wikipedia.org/wiki/Database_model +package lldb + +const ( + fltSz = 0x70 // size of the FLT + maxShort = 251 + maxRq = 65787 + maxFLTRq = 4112 + maxHandle = 1<<56 - 1 + atomLen = 16 + tagUsedLong = 0xfc + tagUsedRelocated = 0xfd + tagFreeShort = 0xfe + tagFreeLong = 0xff + tagNotCompressed = 0 + tagCompressed = 1 +) + +// Content size n -> blocksize in atoms. +func n2atoms(n int) int { + if n > maxShort { + n += 2 + } + return (n+1)/16 + 1 +} + +// Content size n -> number of padding zeros. +func n2padding(n int) int { + if n > maxShort { + n += 2 + } + return 15 - (n+1)&15 +} + +// Handle <-> offset +func h2off(h int64) int64 { return (h + 6) * 16 } +func off2h(off int64) int64 { return off/16 - 6 } + +// Get a 7B int64 from b +func b2h(b []byte) (h int64) { + for _, v := range b[:7] { + h = h<<8 | int64(v) + } + return +} + +// Put a 7B int64 into b +func h2b(b []byte, h int64) []byte { + for i := range b[:7] { + b[i], h = byte(h>>48), h<<8 + } + return b +} + +// Content length N (must be in [252, 65787]) to long used block M field. +func n2m(n int) (m int) { + return n % 0x10000 +} + +// Long used block M (must be in [0, 65535]) field to content length N. +func m2n(m int) (n int) { + if m <= maxShort { + m += 0x10000 + } + return m +} + +func bpack(a []byte) []byte { + if cap(a) > len(a) { + return append([]byte(nil), a...) + } + + return a +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb_test.go new file mode 100644 index 00000000000..408125a5955 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb_test.go @@ -0,0 +1,217 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "bytes" + "encoding/hex" + "fmt" + "path" + "runtime" + "strings" + "testing" +) + +var dbg = func(s string, va ...interface{}) { + _, fn, fl, _ := runtime.Caller(1) + fmt.Printf("%s:%d: ", path.Base(fn), fl) + fmt.Printf(s+"\n", va...) +} + +func use(...interface{}) {} + +func TestN2Atoms(t *testing.T) { + tab := []struct{ n, a int }{ + {0, 1}, + {1, 1}, + {2, 1}, + {3, 1}, + {4, 1}, + {5, 1}, + {6, 1}, + {7, 1}, + {8, 1}, + {9, 1}, + {10, 1}, + {11, 1}, + {12, 1}, + {13, 1}, + {14, 1}, + + {15, 2}, + {16, 2}, + {17, 2}, + {18, 2}, + {19, 2}, + {20, 2}, + {21, 2}, + {22, 2}, + {23, 2}, + {24, 2}, + {25, 2}, + {26, 2}, + {27, 2}, + {28, 2}, + {29, 2}, + {30, 2}, + + {31, 3}, + + {252, 16}, + {253, 17}, + {254, 17}, + {255, 17}, + {256, 17}, + {257, 17}, + {258, 17}, + {259, 17}, + {260, 17}, + {261, 17}, + {262, 17}, + {263, 17}, + {264, 17}, + {265, 17}, + {266, 17}, + {267, 17}, + {268, 17}, + {269, 18}, + {65532, 4096}, + {65533, 4097}, + {65787, 4112}, + } + + for i, test := range tab { + if g, e := n2atoms(test.n), test.a; g != e { + t.Errorf("(%d) %d %d %d", i, test.n, g, e) + } + } +} + +func TestN2Padding(t *testing.T) { + tab := []struct{ n, p int }{ + {0, 14}, + {1, 13}, + {2, 12}, + {3, 11}, + {4, 10}, + {5, 9}, + {6, 8}, + {7, 7}, + {8, 6}, + {9, 5}, + {10, 4}, + {11, 3}, + {12, 2}, + {13, 1}, + {14, 0}, + + {15, 15}, + {16, 14}, + {17, 13}, + {18, 12}, + {19, 11}, + {20, 10}, + {21, 9}, + {22, 8}, + {23, 7}, + {24, 6}, + {25, 5}, + {26, 4}, + {27, 3}, + {28, 2}, + {29, 1}, + {30, 0}, + + {31, 15}, + + {252, 0}, + {253, 15}, + {254, 14}, + {255, 13}, + {256, 12}, + {257, 11}, + {258, 10}, + {259, 9}, + {260, 8}, + {261, 7}, + {262, 6}, + {263, 5}, + {264, 4}, + {265, 3}, + {266, 2}, + {267, 1}, + {268, 0}, + {269, 15}, + } + + for i, test := range tab { + if g, e := n2padding(test.n), test.p; g != e { + t.Errorf("(%d) %d %d %d", i, test.n, g, e) + } + } +} + +func TestH2Off(t *testing.T) { + tab := []struct{ h, off int64 }{ + {-1, fltSz - 32}, + {0, fltSz - 16}, + {1, fltSz + 0}, + {2, fltSz + 16}, + {3, fltSz + 32}, + } + + for i, test := range tab { + if g, e := h2off(test.h), test.off; g != e { + t.Error("h2off", i, g, e) + } + if g, e := off2h(test.off), test.h; g != e { + t.Error("off2h", i, g, e) + } + } +} + +func TestB2H(t *testing.T) { + tab := []struct { + b []byte + h int64 + }{ + {[]byte{0, 0, 0, 0, 0, 0, 0}, 0}, + {[]byte{0, 0, 0, 0, 0, 0, 1}, 1}, + {[]byte{0, 0, 0, 0, 0, 0, 1, 2}, 1}, + {[]byte{0, 0, 0, 0, 0, 0x32, 0x10}, 0x3210}, + {[]byte{0, 0, 0, 0, 0x54, 0x32, 0x10}, 0x543210}, + {[]byte{0, 0, 0, 0x76, 0x54, 0x32, 0x10}, 0x76543210}, + {[]byte{0, 0, 0x98, 0x76, 0x54, 0x32, 0x10}, 0x9876543210}, + {[]byte{0, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, 0xba9876543210}, + {[]byte{0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, 0xdcba9876543210}, + } + + for i, test := range tab { + if g, e := b2h(test.b), test.h; g != e { + t.Errorf("b2h: %d %#8x %#8x", i, g, e) + } + var g [7]byte + h2b(g[:], test.h) + if e := test.b; !bytes.Equal(g[:], e[:7]) { + t.Errorf("b2h: %d g: % 0x e: % 0x", i, g, e) + } + } +} + +func s2b(s string) []byte { + if s == "" { + return nil + } + + s = strings.Replace(s, " ", "", -1) + if n := len(s) & 1; n != 0 { + panic(n) + } + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler.go new file mode 100644 index 00000000000..417e92f3aa2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler.go @@ -0,0 +1,344 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A memory-only implementation of Filer. + +/* + +pgBits: 8 +BenchmarkMemFilerWrSeq 100000 19430 ns/op 1646.93 MB/s +BenchmarkMemFilerRdSeq 100000 17390 ns/op 1840.13 MB/s +BenchmarkMemFilerWrRand 1000000 1903 ns/op 133.94 MB/s +BenchmarkMemFilerRdRand 1000000 1153 ns/op 221.16 MB/s + +pgBits: 9 +BenchmarkMemFilerWrSeq 100000 16195 ns/op 1975.80 MB/s +BenchmarkMemFilerRdSeq 200000 13011 ns/op 2459.39 MB/s +BenchmarkMemFilerWrRand 1000000 2248 ns/op 227.28 MB/s +BenchmarkMemFilerRdRand 1000000 1177 ns/op 433.94 MB/s + +pgBits: 10 +BenchmarkMemFilerWrSeq 100000 16169 ns/op 1979.04 MB/s +BenchmarkMemFilerRdSeq 200000 12673 ns/op 2524.91 MB/s +BenchmarkMemFilerWrRand 1000000 5550 ns/op 184.30 MB/s +BenchmarkMemFilerRdRand 1000000 1699 ns/op 601.79 MB/s + +pgBits: 11 +BenchmarkMemFilerWrSeq 100000 13449 ns/op 2379.31 MB/s +BenchmarkMemFilerRdSeq 200000 12058 ns/op 2653.80 MB/s +BenchmarkMemFilerWrRand 500000 4335 ns/op 471.47 MB/s +BenchmarkMemFilerRdRand 1000000 2843 ns/op 719.47 MB/s + +pgBits: 12 +BenchmarkMemFilerWrSeq 200000 11976 ns/op 2672.00 MB/s +BenchmarkMemFilerRdSeq 200000 12255 ns/op 2611.06 MB/s +BenchmarkMemFilerWrRand 200000 8058 ns/op 507.14 MB/s +BenchmarkMemFilerRdRand 500000 4365 ns/op 936.15 MB/s + +pgBits: 13 +BenchmarkMemFilerWrSeq 200000 10852 ns/op 2948.69 MB/s +BenchmarkMemFilerRdSeq 200000 11561 ns/op 2767.77 MB/s +BenchmarkMemFilerWrRand 200000 9748 ns/op 840.15 MB/s +BenchmarkMemFilerRdRand 500000 7236 ns/op 1131.59 MB/s + +pgBits: 14 +BenchmarkMemFilerWrSeq 200000 10328 ns/op 3098.12 MB/s +BenchmarkMemFilerRdSeq 200000 11292 ns/op 2833.66 MB/s +BenchmarkMemFilerWrRand 100000 16768 ns/op 978.75 MB/s +BenchmarkMemFilerRdRand 200000 13033 ns/op 1258.43 MB/s + +pgBits: 15 +BenchmarkMemFilerWrSeq 200000 10309 ns/op 3103.93 MB/s +BenchmarkMemFilerRdSeq 200000 11126 ns/op 2876.12 MB/s +BenchmarkMemFilerWrRand 50000 31985 ns/op 1021.74 MB/s +BenchmarkMemFilerRdRand 100000 25217 ns/op 1297.65 MB/s + +pgBits: 16 +BenchmarkMemFilerWrSeq 200000 10324 ns/op 3099.45 MB/s +BenchmarkMemFilerRdSeq 200000 11201 ns/op 2856.80 MB/s +BenchmarkMemFilerWrRand 20000 55226 ns/op 1184.76 MB/s +BenchmarkMemFilerRdRand 50000 48316 ns/op 1355.16 MB/s + +pgBits: 17 +BenchmarkMemFilerWrSeq 200000 10377 ns/op 3083.53 MB/s +BenchmarkMemFilerRdSeq 200000 11018 ns/op 2904.18 MB/s +BenchmarkMemFilerWrRand 10000 143425 ns/op 913.12 MB/s +BenchmarkMemFilerRdRand 20000 95267 ns/op 1376.99 MB/s + +pgBits: 18 +BenchmarkMemFilerWrSeq 200000 10312 ns/op 3102.96 MB/s +BenchmarkMemFilerRdSeq 200000 11069 ns/op 2890.84 MB/s +BenchmarkMemFilerWrRand 5000 280910 ns/op 934.14 MB/s +BenchmarkMemFilerRdRand 10000 188500 ns/op 1388.17 MB/s + +*/ + +package lldb + +import ( + "bytes" + "fmt" + "io" + + "github.com/cznic/fileutil" + "github.com/cznic/mathutil" +) + +const ( + pgBits = 16 + pgSize = 1 << pgBits + pgMask = pgSize - 1 +) + +var _ Filer = &MemFiler{} // Ensure MemFiler is a Filer. + +type memFilerMap map[int64]*[pgSize]byte + +// MemFiler is a memory backed Filer. It implements BeginUpdate, EndUpdate and +// Rollback as no-ops. MemFiler is not automatically persistent, but it has +// ReadFrom and WriteTo methods. +type MemFiler struct { + m memFilerMap + nest int + size int64 +} + +// NewMemFiler returns a new MemFiler. +func NewMemFiler() *MemFiler { + return &MemFiler{m: memFilerMap{}} +} + +// BeginUpdate implements Filer. +func (f *MemFiler) BeginUpdate() error { + f.nest++ + return nil +} + +// Close implements Filer. +func (f *MemFiler) Close() (err error) { + if f.nest != 0 { + return &ErrPERM{(f.Name() + ":Close")} + } + + return +} + +// EndUpdate implements Filer. +func (f *MemFiler) EndUpdate() (err error) { + if f.nest == 0 { + return &ErrPERM{(f.Name() + ": EndUpdate")} + } + + f.nest-- + return +} + +// Name implements Filer. +func (f *MemFiler) Name() string { + return fmt.Sprintf("%p.memfiler", f) +} + +// PunchHole implements Filer. +func (f *MemFiler) PunchHole(off, size int64) (err error) { + if off < 0 { + return &ErrINVAL{f.Name() + ": PunchHole off", off} + } + + if size < 0 || off+size > f.size { + return &ErrINVAL{f.Name() + ": PunchHole size", size} + } + + first := off >> pgBits + if off&pgMask != 0 { + first++ + } + off += size - 1 + last := off >> pgBits + if off&pgMask != 0 { + last-- + } + if limit := f.size >> pgBits; last > limit { + last = limit + } + for pg := first; pg <= last; pg++ { + delete(f.m, pg) + } + return +} + +var zeroPage [pgSize]byte + +// ReadAt implements Filer. +func (f *MemFiler) ReadAt(b []byte, off int64) (n int, err error) { + avail := f.size - off + pgI := off >> pgBits + pgO := int(off & pgMask) + rem := len(b) + if int64(rem) >= avail { + rem = int(avail) + err = io.EOF + } + for rem != 0 && avail > 0 { + pg := f.m[pgI] + if pg == nil { + pg = &zeroPage + } + nc := copy(b[:mathutil.Min(rem, pgSize)], pg[pgO:]) + pgI++ + pgO = 0 + rem -= nc + n += nc + b = b[nc:] + } + return +} + +// ReadFrom is a helper to populate MemFiler's content from r. 'n' reports the +// number of bytes read from 'r'. +func (f *MemFiler) ReadFrom(r io.Reader) (n int64, err error) { + if err = f.Truncate(0); err != nil { + return + } + + var ( + b [pgSize]byte + rn int + off int64 + ) + + var rerr error + for rerr == nil { + if rn, rerr = r.Read(b[:]); rn != 0 { + f.WriteAt(b[:rn], off) + off += int64(rn) + n += int64(rn) + } + } + if !fileutil.IsEOF(rerr) { + err = rerr + } + return +} + +// Rollback implements Filer. +func (f *MemFiler) Rollback() (err error) { return } + +// Size implements Filer. +func (f *MemFiler) Size() (int64, error) { + return f.size, nil +} + +// Sync implements Filer. +func (f *MemFiler) Sync() error { + return nil +} + +// Truncate implements Filer. +func (f *MemFiler) Truncate(size int64) (err error) { + switch { + case size < 0: + return &ErrINVAL{"Truncate size", size} + case size == 0: + f.m = memFilerMap{} + f.size = 0 + return + } + + first := size >> pgBits + if size&pgMask != 0 { + first++ + } + last := f.size >> pgBits + if f.size&pgMask != 0 { + last++ + } + for ; first < last; first++ { + delete(f.m, first) + } + + f.size = size + return +} + +// WriteAt implements Filer. +func (f *MemFiler) WriteAt(b []byte, off int64) (n int, err error) { + pgI := off >> pgBits + pgO := int(off & pgMask) + n = len(b) + rem := n + var nc int + for rem != 0 { + if pgO == 0 && rem >= pgSize && bytes.Equal(b[:pgSize], zeroPage[:]) { + delete(f.m, pgI) + nc = pgSize + } else { + pg := f.m[pgI] + if pg == nil { + pg = new([pgSize]byte) + f.m[pgI] = pg + } + nc = copy((*pg)[pgO:], b) + } + pgI++ + pgO = 0 + rem -= nc + b = b[nc:] + } + f.size = mathutil.MaxInt64(f.size, off+int64(n)) + return +} + +// WriteTo is a helper to copy/persist MemFiler's content to w. If w is also +// an io.WriterAt then WriteTo may attempt to _not_ write any big, for some +// value of big, runs of zeros, i.e. it will attempt to punch holes, where +// possible, in `w` if that happens to be a freshly created or to zero length +// truncated OS file. 'n' reports the number of bytes written to 'w'. +func (f *MemFiler) WriteTo(w io.Writer) (n int64, err error) { + var ( + b [pgSize]byte + wn, rn int + off int64 + rerr error + ) + + if wa, ok := w.(io.WriterAt); ok { + lastPgI := f.size >> pgBits + for pgI := int64(0); pgI <= lastPgI; pgI++ { + sz := pgSize + if pgI == lastPgI { + sz = int(f.size & pgMask) + } + pg := f.m[pgI] + if pg != nil { + wn, err = wa.WriteAt(pg[:sz], off) + if err != nil { + return + } + + n += int64(wn) + off += int64(sz) + if wn != sz { + return n, io.ErrShortWrite + } + } + } + return + } + + var werr error + for rerr == nil { + if rn, rerr = f.ReadAt(b[:], off); rn != 0 { + off += int64(rn) + if wn, werr = w.Write(b[:rn]); werr != nil { + return n, werr + } + + n += int64(wn) + } + } + if !fileutil.IsEOF(rerr) { + err = rerr + } + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler_test.go new file mode 100644 index 00000000000..319f4bbbf8a --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler_test.go @@ -0,0 +1,132 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "bytes" + "math/rand" + "testing" +) + +// Test automatic page releasing (hole punching) of zero pages +func TestMemFilerWriteAt(t *testing.T) { + f := NewMemFiler() + + // Add page index 0 + if _, err := f.WriteAt([]byte{1}, 0); err != nil { + t.Fatal(err) + } + + if g, e := len(f.m), 1; g != e { + t.Fatal(g, e) + } + + // Add page index 1 + if _, err := f.WriteAt([]byte{2}, pgSize); err != nil { + t.Fatal(err) + } + + if g, e := len(f.m), 2; g != e { + t.Fatal(g, e) + } + + // Add page index 2 + if _, err := f.WriteAt([]byte{3}, 2*pgSize); err != nil { + t.Fatal(err) + } + + if g, e := len(f.m), 3; g != e { + t.Fatal(g, e) + } + + // Remove page index 1 + if _, err := f.WriteAt(make([]byte, 2*pgSize), pgSize/2); err != nil { + t.Fatal(err) + } + + if g, e := len(f.m), 2; g != e { + t.Logf("%#v", f.m) + t.Fatal(g, e) + } + + if err := f.Truncate(1); err != nil { + t.Fatal(err) + } + + if g, e := len(f.m), 1; g != e { + t.Logf("%#v", f.m) + t.Fatal(g, e) + } + + if err := f.Truncate(0); err != nil { + t.Fatal(err) + } + + if g, e := len(f.m), 0; g != e { + t.Logf("%#v", f.m) + t.Fatal(g, e) + } +} + +func TestMemFilerWriteTo(t *testing.T) { + const max = 1e5 + var b [max]byte + rng := rand.New(rand.NewSource(42)) + for sz := 0; sz < 1e5; sz += 2053 { + for i := range b[:sz] { + b[i] = byte(rng.Int()) + } + f := NewMemFiler() + if n, err := f.WriteAt(b[:sz], 0); n != sz || err != nil { + t.Fatal(n, err) + } + + var buf bytes.Buffer + if n, err := f.WriteTo(&buf); n != int64(sz) || err != nil { + t.Fatal(n, err) + } + + if !bytes.Equal(b[:sz], buf.Bytes()) { + t.Fatal("content differs") + } + } +} + +func TestMemFilerReadFromWriteTo(t *testing.T) { + const ( + sz = 1e2 * pgSize + hole = 1e1 * pgSize + ) + rng := rand.New(rand.NewSource(42)) + data := make([]byte, sz) + for i := range data { + data[i] = byte(rng.Int()) + } + f := NewMemFiler() + buf := bytes.NewBuffer(data) + if n, err := f.ReadFrom(buf); n != int64(len(data)) || err != nil { + t.Fatal(n, err) + } + + buf = bytes.NewBuffer(nil) + if n, err := f.WriteTo(buf); n != int64(len(data)) || err != nil { + t.Fatal(n, err) + } + + rd := buf.Bytes() + if !bytes.Equal(data, rd) { + t.Fatal("corrupted data") + } + + n0 := len(f.m) + data = make([]byte, hole) + f.WriteAt(data, sz/2) + n := len(f.m) + t.Log(n0, n) + d := n0 - n + if d*pgSize < hole-2 || d*pgSize > hole { + t.Fatal(n0, n, d) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/osfiler.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/osfiler.go new file mode 100644 index 00000000000..d6e189a18f4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/osfiler.go @@ -0,0 +1,130 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "io" + "os" + + "github.com/cznic/mathutil" +) + +var _ Filer = (*OSFiler)(nil) + +// OSFile is an os.File like minimal set of methods allowing to construct a +// Filer. +type OSFile interface { + Name() string + Stat() (fi os.FileInfo, err error) + Sync() (err error) + Truncate(size int64) (err error) + io.Closer + io.Reader + io.ReaderAt + io.Seeker + io.Writer + io.WriterAt +} + +// OSFiler is like a SimpleFileFiler but based on an OSFile. +type OSFiler struct { + f OSFile + nest int + size int64 // not set if < 0 +} + +// NewOSFiler returns a Filer from an OSFile. This Filer is like the +// SimpleFileFiler, it does not implement the transaction related methods. +func NewOSFiler(f OSFile) (r *OSFiler) { + return &OSFiler{ + f: f, + size: -1, + } +} + +// BeginUpdate implements Filer. +func (f *OSFiler) BeginUpdate() (err error) { + f.nest++ + return nil +} + +// Close implements Filer. +func (f *OSFiler) Close() (err error) { + if f.nest != 0 { + return &ErrPERM{(f.Name() + ":Close")} + } + + return f.f.Close() +} + +// EndUpdate implements Filer. +func (f *OSFiler) EndUpdate() (err error) { + if f.nest == 0 { + return &ErrPERM{(f.Name() + ":EndUpdate")} + } + + f.nest-- + return +} + +// Name implements Filer. +func (f *OSFiler) Name() string { + return f.f.Name() +} + +// PunchHole implements Filer. +func (f *OSFiler) PunchHole(off, size int64) (err error) { + return +} + +// ReadAt implements Filer. +func (f *OSFiler) ReadAt(b []byte, off int64) (n int, err error) { + return f.f.ReadAt(b, off) +} + +// Rollback implements Filer. +func (f *OSFiler) Rollback() (err error) { return } + +// Size implements Filer. +func (f *OSFiler) Size() (n int64, err error) { + if f.size < 0 { // boot + fi, err := f.f.Stat() + if err != nil { + return 0, err + } + + f.size = fi.Size() + } + return f.size, nil +} + +// Sync implements Filer. +func (f *OSFiler) Sync() (err error) { + return f.f.Sync() +} + +// Truncate implements Filer. +func (f *OSFiler) Truncate(size int64) (err error) { + if size < 0 { + return &ErrINVAL{"Truncate size", size} + } + + f.size = size + return f.f.Truncate(size) +} + +// WriteAt implements Filer. +func (f *OSFiler) WriteAt(b []byte, off int64) (n int, err error) { + if f.size < 0 { // boot + fi, err := os.Stat(f.f.Name()) + if err != nil { + return 0, err + } + + f.size = fi.Size() + } + f.size = mathutil.MaxInt64(f.size, int64(len(b))+off) + return f.f.WriteAt(b, off) +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/simplefilefiler.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/simplefilefiler.go new file mode 100644 index 00000000000..de32e649166 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/simplefilefiler.go @@ -0,0 +1,123 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A basic os.File backed Filer. + +package lldb + +import ( + "os" + + "github.com/cznic/fileutil" + "github.com/cznic/mathutil" +) + +var _ Filer = &SimpleFileFiler{} // Ensure SimpleFileFiler is a Filer. + +// SimpleFileFiler is an os.File backed Filer intended for use where structural +// consistency can be reached by other means (SimpleFileFiler is for example +// wrapped in eg. an RollbackFiler or ACIDFiler0) or where persistence is not +// required (temporary/working data sets). +// +// SimpleFileFiler is the most simple os.File backed Filer implementation as it +// does not really implement BeginUpdate and EndUpdate/Rollback in any way +// which would protect the structural integrity of data. If misused e.g. as a +// real database storage w/o other measures, it can easily cause data loss +// when, for example, a power outage occurs or the updating process terminates +// abruptly. +type SimpleFileFiler struct { + file *os.File + nest int + size int64 // not set if < 0 +} + +// NewSimpleFileFiler returns a new SimpleFileFiler. +func NewSimpleFileFiler(f *os.File) *SimpleFileFiler { + return &SimpleFileFiler{file: f, size: -1} +} + +// BeginUpdate implements Filer. +func (f *SimpleFileFiler) BeginUpdate() error { + f.nest++ + return nil +} + +// Close implements Filer. +func (f *SimpleFileFiler) Close() (err error) { + if f.nest != 0 { + return &ErrPERM{(f.Name() + ":Close")} + } + + return f.file.Close() +} + +// EndUpdate implements Filer. +func (f *SimpleFileFiler) EndUpdate() (err error) { + if f.nest == 0 { + return &ErrPERM{(f.Name() + ":EndUpdate")} + } + + f.nest-- + return +} + +// Name implements Filer. +func (f *SimpleFileFiler) Name() string { + return f.file.Name() +} + +// PunchHole implements Filer. +func (f *SimpleFileFiler) PunchHole(off, size int64) (err error) { + return fileutil.PunchHole(f.file, off, size) +} + +// ReadAt implements Filer. +func (f *SimpleFileFiler) ReadAt(b []byte, off int64) (n int, err error) { + return f.file.ReadAt(b, off) +} + +// Rollback implements Filer. +func (f *SimpleFileFiler) Rollback() (err error) { return } + +// Size implements Filer. +func (f *SimpleFileFiler) Size() (int64, error) { + if f.size < 0 { // boot + fi, err := os.Stat(f.file.Name()) + if err != nil { + return 0, err + } + + f.size = fi.Size() + } + return f.size, nil +} + +// Sync implements Filer. +func (f *SimpleFileFiler) Sync() error { + return f.file.Sync() +} + +// Truncate implements Filer. +func (f *SimpleFileFiler) Truncate(size int64) (err error) { + if size < 0 { + return &ErrINVAL{"Truncate size", size} + } + + f.size = size + return f.file.Truncate(size) +} + +// WriteAt implements Filer. +func (f *SimpleFileFiler) WriteAt(b []byte, off int64) (n int, err error) { + if f.size < 0 { // boot + fi, err := os.Stat(f.file.Name()) + if err != nil { + return 0, err + } + + f.size = fi.Size() + } + f.size = mathutil.MaxInt64(f.size, int64(len(b))+off) + return f.file.WriteAt(b, off) +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/xact.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/xact.go new file mode 100644 index 00000000000..f9ad1cbfb70 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/xact.go @@ -0,0 +1,629 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Structural transactions. + +package lldb + +//DONE+ TransactionalMemoryFiler +// ---- +// Use NewRollbackFiler(myMemFiler, ...) + +/* + +bfBits: 3 +BenchmarkRollbackFiler 20000000 102 ns/op 9.73 MB/s + +bfBits: 4 +BenchmarkRollbackFiler 50000000 55.7 ns/op 17.95 MB/s + +bfBits: 5 +BenchmarkRollbackFiler 100000000 32.2 ns/op 31.06 MB/s + +bfBits: 6 +BenchmarkRollbackFiler 100000000 20.6 ns/op 48.46 MB/s + +bfBits: 7 +BenchmarkRollbackFiler 100000000 15.1 ns/op 66.12 MB/s + +bfBits: 8 +BenchmarkRollbackFiler 100000000 10.5 ns/op 95.66 MB/s + +bfBits: 9 +BenchmarkRollbackFiler 200000000 8.02 ns/op 124.74 MB/s + +bfBits: 10 +BenchmarkRollbackFiler 200000000 9.25 ns/op 108.09 MB/s + +bfBits: 11 +BenchmarkRollbackFiler 100000000 11.7 ns/op 85.47 MB/s + +bfBits: 12 +BenchmarkRollbackFiler 100000000 17.2 ns/op 57.99 MB/s + +bfBits: 13 +BenchmarkRollbackFiler 100000000 32.7 ns/op 30.58 MB/s + +bfBits: 14 +BenchmarkRollbackFiler 50000000 39.6 ns/op 25.27 MB/s + +*/ + +import ( + "fmt" + "io" + "sync" + + "github.com/cznic/fileutil" + "github.com/cznic/mathutil" +) + +var ( + _ Filer = &bitFiler{} // Ensure bitFiler is a Filer. + _ Filer = &RollbackFiler{} // ditto +) + +const ( + bfBits = 9 + bfSize = 1 << bfBits + bfMask = bfSize - 1 +) + +var ( + bitmask = [8]byte{1, 2, 4, 8, 16, 32, 64, 128} + bitZeroPage bitPage + allDirtyFlags [bfSize >> 3]byte +) + +func init() { + for i := range allDirtyFlags { + allDirtyFlags[i] = 0xff + } +} + +type ( + bitPage struct { + prev, next *bitPage + data [bfSize]byte + flags [bfSize >> 3]byte + dirty bool + } + + bitFilerMap map[int64]*bitPage + + bitFiler struct { + parent Filer + m bitFilerMap + size int64 + } +) + +func newBitFiler(parent Filer) (f *bitFiler, err error) { + sz, err := parent.Size() + if err != nil { + return + } + + return &bitFiler{parent: parent, m: bitFilerMap{}, size: sz}, nil +} + +func (f *bitFiler) BeginUpdate() error { panic("internal error") } +func (f *bitFiler) EndUpdate() error { panic("internal error") } +func (f *bitFiler) Rollback() error { panic("internal error") } +func (f *bitFiler) Sync() error { panic("internal error") } + +func (f *bitFiler) Close() (err error) { return } +func (f *bitFiler) Name() string { return fmt.Sprintf("%p.bitfiler", f) } +func (f *bitFiler) Size() (int64, error) { return f.size, nil } + +func (f *bitFiler) PunchHole(off, size int64) (err error) { + first := off >> bfBits + if off&bfMask != 0 { + first++ + } + off += size - 1 + last := off >> bfBits + if off&bfMask != 0 { + last-- + } + if limit := f.size >> bfBits; last > limit { + last = limit + } + for pgI := first; pgI <= last; pgI++ { + pg := &bitPage{} + pg.flags = allDirtyFlags + f.m[pgI] = pg + } + return +} + +func (f *bitFiler) ReadAt(b []byte, off int64) (n int, err error) { + avail := f.size - off + pgI := off >> bfBits + pgO := int(off & bfMask) + rem := len(b) + if int64(rem) >= avail { + rem = int(avail) + err = io.EOF + } + for rem != 0 && avail > 0 { + pg := f.m[pgI] + if pg == nil { + pg = &bitPage{} + if f.parent != nil { + _, err = f.parent.ReadAt(pg.data[:], off&^bfMask) + if err != nil && !fileutil.IsEOF(err) { + return + } + + err = nil + } + f.m[pgI] = pg + } + nc := copy(b[:mathutil.Min(rem, bfSize)], pg.data[pgO:]) + pgI++ + pgO = 0 + rem -= nc + n += nc + b = b[nc:] + off += int64(nc) + } + return +} + +func (f *bitFiler) Truncate(size int64) (err error) { + switch { + case size < 0: + return &ErrINVAL{"Truncate size", size} + case size == 0: + f.m = bitFilerMap{} + f.size = 0 + return + } + + first := size >> bfBits + if size&bfMask != 0 { + first++ + } + last := f.size >> bfBits + if f.size&bfMask != 0 { + last++ + } + for ; first < last; first++ { + delete(f.m, first) + } + + f.size = size + return +} + +func (f *bitFiler) WriteAt(b []byte, off int64) (n int, err error) { + off0 := off + pgI := off >> bfBits + pgO := int(off & bfMask) + n = len(b) + rem := n + var nc int + for rem != 0 { + pg := f.m[pgI] + if pg == nil { + pg = &bitPage{} + if f.parent != nil { + _, err = f.parent.ReadAt(pg.data[:], off&^bfMask) + if err != nil && !fileutil.IsEOF(err) { + return + } + + err = nil + } + f.m[pgI] = pg + } + nc = copy(pg.data[pgO:], b) + pgI++ + pg.dirty = true + for i := pgO; i < pgO+nc; i++ { + pg.flags[i>>3] |= bitmask[i&7] + } + pgO = 0 + rem -= nc + b = b[nc:] + off += int64(nc) + } + f.size = mathutil.MaxInt64(f.size, off0+int64(n)) + return +} + +func (f *bitFiler) link() { + for pgI, pg := range f.m { + nx, ok := f.m[pgI+1] + if !ok || !nx.dirty { + continue + } + + nx.prev, pg.next = pg, nx + } +} + +func (f *bitFiler) dumpDirty(w io.WriterAt) (nwr int, err error) { + f.link() + for pgI, pg := range f.m { + if !pg.dirty { + continue + } + + for pg.prev != nil && pg.prev.dirty { + pg = pg.prev + pgI-- + } + + for pg != nil && pg.dirty { + last := false + var off int64 + first := -1 + for i := 0; i < bfSize; i++ { + flag := pg.flags[i>>3]&bitmask[i&7] != 0 + switch { + case flag && !last: // Leading edge detected + off = pgI<= 0 { + i := bfSize + n, err := w.WriteAt(pg.data[first:i], off) + if n != i-first { + return 0, err + } + + nwr++ + } + + pg.dirty = false + pg = pg.next + pgI++ + } + } + return +} + +// RollbackFiler is a Filer implementing structural transaction handling. +// Structural transactions should be small and short lived because all non +// committed data are held in memory until committed or discarded by a +// Rollback. +// +// While using RollbackFiler, every intended update of the wrapped Filler, by +// WriteAt, Truncate or PunchHole, _must_ be made within a transaction. +// Attempts to do it outside of a transaction will return ErrPERM. OTOH, +// invoking ReadAt outside of a transaction is not a problem. +// +// No nested transactions: All updates within a transaction are held in memory. +// On a matching EndUpdate the updates held in memory are actually written to +// the wrapped Filer. +// +// Nested transactions: Correct data will be seen from RollbackFiler when any +// level of a nested transaction is rollbacked. The actual writing to the +// wrapped Filer happens only when the outer most transaction nesting level is +// closed. +// +// Invoking Rollback is an alternative to EndUpdate. It discards all changes +// made at the current transaction level and returns the "state" (possibly not +// yet persisted) of the Filer to what it was before the corresponding +// BeginUpdate. +// +// During an open transaction, all reads (using ReadAt) are "dirty" reads, +// seeing the uncommitted changes made to the Filer's data. +// +// Lldb databases should be based upon a RollbackFiler. +// +// With a wrapped MemFiler one gets transactional memory. With, for example a +// wrapped disk based SimpleFileFiler it protects against at least some HW +// errors - if Rollback is properly invoked on such failures and/or if there's +// some WAL or 2PC or whatever other safe mechanism based recovery procedure +// used by the client. +// +// The "real" writes to the wrapped Filer (or WAL instead) go through the +// writerAt supplied to NewRollbackFiler. +// +// List of functions/methods which are recommended to be wrapped in a +// BeginUpdate/EndUpdate structural transaction: +// +// Allocator.Alloc +// Allocator.Free +// Allocator.Realloc +// +// CreateBTree +// RemoveBTree +// BTree.Clear +// BTree.Delete +// BTree.DeleteAny +// BTree.Clear +// BTree.Extract +// BTree.Get (it can mutate the DB) +// BTree.Put +// BTree.Set +// +// NOTE: RollbackFiler is a generic solution intended to wrap Filers provided +// by this package which do not implement any of the transactional methods. +// RollbackFiler thus _does not_ invoke any of the transactional methods of its +// wrapped Filer. +// +// RollbackFiler is safe for concurrent use by multiple goroutines. +type RollbackFiler struct { + mu sync.RWMutex + inCallback bool + inCallbackMu sync.RWMutex + bitFiler *bitFiler + checkpoint func(int64) error + closed bool + f Filer + parent Filer + tlevel int // transaction nesting level, 0 == not in transaction + writerAt io.WriterAt + + // afterRollback, if not nil, is called after performing Rollback + // without errros. + afterRollback func() error +} + +// NewRollbackFiler returns a RollbackFiler wrapping f. +// +// The checkpoint parameter +// +// The checkpoint function is called after closing (by EndUpdate) the upper +// most level open transaction if all calls of writerAt were successful and the +// DB (or eg. a WAL) is thus now in a consistent state (virtually, in the ideal +// world with no write caches, no HW failures, no process crashes, ...). +// +// NOTE: In, for example, a 2PC it is necessary to reflect also the sz +// parameter as the new file size (as in the parameter to Truncate). All +// changes were successfully written already by writerAt before invoking +// checkpoint. +// +// The writerAt parameter +// +// The writerAt interface is used to commit the updates of the wrapped Filer. +// If any invocation of writerAt fails then a non nil error will be returned +// from EndUpdate and checkpoint will _not_ ne called. Neither is necessary to +// call Rollback. The rule of thumb: The [structural] transaction [level] is +// closed by invoking exactly once one of EndUpdate _or_ Rollback. +// +// It is presumed that writerAt uses WAL or 2PC or whatever other safe +// mechanism to physically commit the updates. +// +// Updates performed by invocations of writerAt are byte-precise, but not +// necessarily maximum possible length precise. IOW, for example an update +// crossing page boundaries may be performed by more than one writerAt +// invocation. No offset sorting is performed. This may change if it proves +// to be a problem. Such change would be considered backward compatible. +// +// NOTE: Using RollbackFiler, but failing to ever invoke a matching "closing" +// EndUpdate after an "opening" BeginUpdate means neither writerAt or +// checkpoint will ever get called - with all the possible data loss +// consequences. +func NewRollbackFiler(f Filer, checkpoint func(sz int64) error, writerAt io.WriterAt) (r *RollbackFiler, err error) { + if f == nil || checkpoint == nil || writerAt == nil { + return nil, &ErrINVAL{Src: "lldb.NewRollbackFiler, nil argument"} + } + + return &RollbackFiler{ + checkpoint: checkpoint, + f: f, + writerAt: writerAt, + }, nil +} + +// Implements Filer. +func (r *RollbackFiler) BeginUpdate() (err error) { + r.mu.Lock() + defer r.mu.Unlock() + + parent := r.f + if r.tlevel != 0 { + parent = r.bitFiler + } + r.bitFiler, err = newBitFiler(parent) + if err != nil { + return + } + + r.tlevel++ + return +} + +// Implements Filer. +// +// Close will return an error if not invoked at nesting level 0. However, to +// allow emergency closing from eg. a signal handler; if Close is invoked +// within an open transaction(s), it rollbacks any non committed open +// transactions and performs the Close operation. +// +// IOW: Regardless of the transaction nesting level the Close is always +// performed but any uncommitted transaction data are lost. +func (r *RollbackFiler) Close() (err error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.closed { + return &ErrPERM{r.f.Name() + ": Already closed"} + } + + r.closed = true + if err = r.f.Close(); err != nil { + return + } + + if r.tlevel != 0 { + err = &ErrPERM{r.f.Name() + ": Close inside an open transaction"} + } + + return +} + +// Implements Filer. +func (r *RollbackFiler) EndUpdate() (err error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.tlevel == 0 { + return &ErrPERM{r.f.Name() + " : EndUpdate outside of a transaction"} + } + + sz, err := r.size() // Cannot call .Size() -> deadlock + if err != nil { + return + } + + r.tlevel-- + bf := r.bitFiler + parent := bf.parent + w := r.writerAt + if r.tlevel != 0 { + w = parent + } + nwr, err := bf.dumpDirty(w) + if err != nil { + return + } + + switch { + case r.tlevel == 0: + r.bitFiler = nil + if nwr == 0 { + return + } + + return r.checkpoint(sz) + default: + r.bitFiler = parent.(*bitFiler) + sz, _ := bf.Size() // bitFiler.Size() never returns err != nil + return parent.Truncate(sz) + } +} + +// Implements Filer. +func (r *RollbackFiler) Name() string { + r.mu.RLock() + defer r.mu.RUnlock() + + return r.f.Name() +} + +// Implements Filer. +func (r *RollbackFiler) PunchHole(off, size int64) error { + r.mu.Lock() + defer r.mu.Unlock() + + if r.tlevel == 0 { + return &ErrPERM{r.f.Name() + ": PunchHole outside of a transaction"} + } + + if off < 0 { + return &ErrINVAL{r.f.Name() + ": PunchHole off", off} + } + + if size < 0 || off+size > r.bitFiler.size { + return &ErrINVAL{r.f.Name() + ": PunchHole size", size} + } + + return r.bitFiler.PunchHole(off, size) +} + +// Implements Filer. +func (r *RollbackFiler) ReadAt(b []byte, off int64) (n int, err error) { + r.inCallbackMu.RLock() + defer r.inCallbackMu.RUnlock() + if !r.inCallback { + r.mu.RLock() + defer r.mu.RUnlock() + } + if r.tlevel == 0 { + return r.f.ReadAt(b, off) + } + + return r.bitFiler.ReadAt(b, off) +} + +// Implements Filer. +func (r *RollbackFiler) Rollback() (err error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.tlevel == 0 { + return &ErrPERM{r.f.Name() + ": Rollback outside of a transaction"} + } + + if r.tlevel > 1 { + r.bitFiler = r.bitFiler.parent.(*bitFiler) + } + r.tlevel-- + if f := r.afterRollback; f != nil { + r.inCallbackMu.Lock() + r.inCallback = true + r.inCallbackMu.Unlock() + defer func() { + r.inCallbackMu.Lock() + r.inCallback = false + r.inCallbackMu.Unlock() + }() + return f() + } + return +} + +func (r *RollbackFiler) size() (sz int64, err error) { + if r.tlevel == 0 { + return r.f.Size() + } + + return r.bitFiler.Size() +} + +// Implements Filer. +func (r *RollbackFiler) Size() (sz int64, err error) { + r.mu.Lock() + defer r.mu.Unlock() + + return r.size() +} + +// Implements Filer. +func (r *RollbackFiler) Sync() error { + r.mu.Lock() + defer r.mu.Unlock() + + return r.f.Sync() +} + +// Implements Filer. +func (r *RollbackFiler) Truncate(size int64) error { + r.mu.Lock() + defer r.mu.Unlock() + + if r.tlevel == 0 { + return &ErrPERM{r.f.Name() + ": Truncate outside of a transaction"} + } + + return r.bitFiler.Truncate(size) +} + +// Implements Filer. +func (r *RollbackFiler) WriteAt(b []byte, off int64) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.tlevel == 0 { + return 0, &ErrPERM{r.f.Name() + ": WriteAt outside of a transaction"} + } + + return r.bitFiler.WriteAt(b, off) +} diff --git a/Godeps/_workspace/src/github.com/cznic/exp/lldb/xact_test.go b/Godeps/_workspace/src/github.com/cznic/exp/lldb/xact_test.go new file mode 100644 index 00000000000..77b204d9072 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/exp/lldb/xact_test.go @@ -0,0 +1,400 @@ +// Copyright 2014 The lldb Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lldb + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "math/rand" + "testing" + + "github.com/cznic/fileutil" + "github.com/cznic/mathutil" +) + +func (f *bitFiler) dump(w io.Writer) { + fmt.Fprintf(w, "bitFiler @ %p, size: %d(%#x)\n", f, f.size, f.size) + for k, v := range f.m { + fmt.Fprintf(w, "bitPage @ %p: pgI %d(%#x): %#v\n", v, k, k, *v) + } +} + +func filerBytes(f Filer) []byte { + sz, err := f.Size() + if err != nil { + panic(err) + } + + b := make([]byte, int(sz)) + n, err := f.ReadAt(b, 0) + if n != len(b) { + panic(fmt.Errorf("sz %d n %d err %v", sz, n, err)) + } + + return b +} + +func cmpFilerBytes(t *testing.T, fg, fe Filer) { + g, e := filerBytes(fg), filerBytes(fe) + if !bytes.Equal(g, e) { + t.Fatalf("Filer content doesn't match: got\n%sexp:\n%s", hex.Dump(g), hex.Dump(e)) + } +} + +func TestRollbackFiler0(t *testing.T) { + var r *RollbackFiler + f, g := NewMemFiler(), NewMemFiler() + + checkpoint := func(sz int64) (err error) { + return f.Truncate(sz) + } + + r, err := NewRollbackFiler(f, checkpoint, f) + if err != nil { + t.Fatal(err) + } + + if err = r.BeginUpdate(); err != nil { + t.Fatal(err) + } + + if err = r.EndUpdate(); err != nil { + t.Fatal(err) + } + + cmpFilerBytes(t, f, g) +} + +func TestRollbackFiler1(t *testing.T) { + const ( + N = 1e6 + O = 1234 + ) + + var r *RollbackFiler + f, g := NewMemFiler(), NewMemFiler() + + checkpoint := func(sz int64) (err error) { + return f.Truncate(sz) + } + + r, err := NewRollbackFiler(f, checkpoint, f) + if err != nil { + t.Fatal(err) + } + + if err = r.BeginUpdate(); err != nil { + t.Fatal(err) + } + + rng := rand.New(rand.NewSource(42)) + b := make([]byte, N) + for i := range b { + b[i] = byte(rng.Int()) + } + + if _, err = g.WriteAt(b, O); err != nil { + t.Fatal(err) + } + + if _, err = r.WriteAt(b, O); err != nil { + t.Fatal(err) + } + + b = filerBytes(f) + if n := len(b); n != 0 { + t.Fatal(n) + } + + if err = r.EndUpdate(); err != nil { + t.Fatal(err) + } + + cmpFilerBytes(t, f, g) +} + +func TestRollbackFiler2(t *testing.T) { + const ( + N = 1e6 + O = 1234 + ) + + var r *RollbackFiler + f, g := NewMemFiler(), NewMemFiler() + + checkpoint := func(sz int64) (err error) { + return f.Truncate(sz) + } + + r, err := NewRollbackFiler(f, checkpoint, f) + if err != nil { + t.Fatal(err) + } + + if err = r.BeginUpdate(); err != nil { + t.Fatal(err) + } + + rng := rand.New(rand.NewSource(42)) + b := make([]byte, N) + for i := range b { + b[i] = byte(rng.Int()) + } + + if _, err = r.WriteAt(b, O); err != nil { + t.Fatal(err) + } + + b = filerBytes(f) + if n := len(b); n != 0 { + t.Fatal(n) + } + + if err = r.Rollback(); err != nil { + t.Fatal(err) + } + + cmpFilerBytes(t, f, g) +} + +func rndBytes(rng *rand.Rand, n int) []byte { + r := make([]byte, n) + for i := range r { + r[i] = byte(rng.Int()) + } + return r +} + +func TestRollbackFiler3(t *testing.T) { + var r *RollbackFiler + f := NewMemFiler() + + checkpoint := func(sz int64) (err error) { + return f.Truncate(sz) + } + + r, err := NewRollbackFiler(f, checkpoint, f) + if err != nil { + t.Fatal(err) + } + + n, err := r.ReadAt([]byte{0}, 0) + if n != 0 || !fileutil.IsEOF(err) { + t.Fatal(n, err) + } + + n, err = r.ReadAt([]byte{0}, 1e6) + if n != 0 || !fileutil.IsEOF(err) { + t.Fatal(n, err) + } + + if err = r.BeginUpdate(); err != nil { // BeginUpdate: 0 -> 1 + t.Fatal(err) + } + + rng := rand.New(rand.NewSource(42)) + + buf := rndBytes(rng, 100) + if n, err := r.WriteAt(buf, 1e6); n != 100 || err != nil { + t.Fatal(err) + } + + buf = make([]byte, 100) + if n, err := r.ReadAt(buf, 1e6-200); n != 100 || err != nil { + t.Fatal(err) + } + + for i, v := range buf { + if v != 0 { + t.Fatal(i, v) + } + } + + if err := r.Truncate(1e5); err != nil { + t.Fatal(err) + } + + if err = r.BeginUpdate(); err != nil { // BeginUpdate: 1 -> 2 + t.Fatal(err) + } + + if n, err := r.ReadAt(buf, 1e6); n != 0 || err == nil { + t.Fatal(n, err) + } + + if err := r.Truncate(2e6); err != nil { + t.Fatal(err) + } + + if err = r.BeginUpdate(); err != nil { // BeginUpdate: 2 -> 3 + t.Fatal(err) + } + + if n, err := r.ReadAt(buf, 1e6); n == 0 || err != nil { + t.Fatal(n, err) + } + + for i, v := range buf { + if v != 0 { + t.Fatal(i, v) + } + } +} + +func TestRollbackFiler4(t *testing.T) { + const ( + maxSize = 1e6 + maxChange = maxSize/100 + 4 + maxChanges = 10 + maxNest = 3 + ) + + var r *RollbackFiler + f := NewMemFiler() + + checkpoint := func(sz int64) (err error) { + return f.Truncate(sz) + } + + r, err := NewRollbackFiler(f, checkpoint, f) + if err != nil { + t.Fatal(err) + } + + rng := rand.New(rand.NewSource(42)) + + ref := make([]byte, 2*maxSize) + for i := range ref { + ref[i] = byte(rng.Int()) + } + + var finalSize int + + var fn func(int, int, []byte) (int, []byte) + fn = func(nest, inSize int, in []byte) (outSize int, out []byte) { + defer func() { + for i := outSize; i < len(out); i++ { + out[i] = 0 + } + finalSize = mathutil.Max(finalSize, outSize) + }() + + out = make([]byte, len(in), 2*maxSize) + copy(out, in) + if err := r.BeginUpdate(); err != nil { + t.Fatal(err) + } + + for i := 0; i < maxChanges; i++ { + changeLen := rng.Intn(maxChange) + 4 + changeOff := rng.Intn(maxSize * 3 / 2) + b := make([]byte, changeLen) + for i := range b { + b[i] = byte(rng.Int()) + } + if n, err := r.WriteAt(b, int64(changeOff)); n != len(b) || err != nil { + t.Fatal(n, len(b), err) + } + } + + if err := r.Rollback(); err != nil { + t.Fatal(err) + } + + if err := r.BeginUpdate(); err != nil { + t.Fatal(err) + } + + for i := 0; i < maxChanges; i++ { + changeLen := rng.Intn(maxChange) + 4 + changeOff := rng.Intn(maxSize * 3 / 2) + b := make([]byte, changeLen) + for i := range b { + b[i] = byte(rng.Int()) + } + if n, err := r.WriteAt(b, int64(changeOff)); n != len(b) || err != nil { + t.Fatal(n, len(b), err) + } + copy(out[changeOff:], b) + copy(ref[changeOff:], b) + } + + newSize := rng.Intn(maxSize*3/2) + 4 + if nest == maxNest { + if err := r.EndUpdate(); err != nil { + t.Fatal(err) + } + + return newSize, out + } + + outSize, out = fn(nest+1, newSize, out) + if err := r.EndUpdate(); err != nil { + t.Fatal(err) + } + + return + } + + sz, result := fn(0, maxSize, ref) + if g, e := sz, finalSize; g != e { + t.Fatal(err) + } + + g, e := result[:sz], ref[:sz] + if !bytes.Equal(g, e) { + if len(g) == len(e) { + x := make([]byte, len(g)) + for i := range x { + if g[i] != e[i] { + x[i] = 'X' + } + } + //t.Logf("Data diff\n%s", hex.Dump(x)) + } + //t.Fatalf("Data don't match: got\n%sexp:\n%s", hex.Dump(g), hex.Dump(e)) + t.Fatalf("Data don't match") + } +} + +func BenchmarkRollbackFiler(b *testing.B) { + rng := rand.New(rand.NewSource(42)) + type t struct { + off int64 + b []byte + } + a := []t{} + for rem := b.N; rem > 0; { + off := rng.Int63() + n := mathutil.Min(rng.Intn(1e3)+1, rem) + a = append(a, t{off, rndBytes(rng, n)}) + rem -= n + } + + var r *RollbackFiler + f := NewMemFiler() + + checkpoint := func(sz int64) (err error) { + return f.Truncate(sz) + } + + r, err := NewRollbackFiler(f, checkpoint, f) + if err != nil { + b.Fatal(err) + } + + if err := r.BeginUpdate(); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for _, v := range a { + if _, err := r.WriteAt(v.b, v.off); err != nil { + b.Fatal(err) + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/AUTHORS b/Godeps/_workspace/src/github.com/cznic/fileutil/AUTHORS new file mode 100644 index 00000000000..288ea0b1e79 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/AUTHORS @@ -0,0 +1,14 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +CZ.NIC z.s.p.o. +Jan Mercl <0xjnml@gmail.com> +Aaron Bieber + diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/fileutil/CONTRIBUTORS new file mode 100644 index 00000000000..223b21a3074 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/CONTRIBUTORS @@ -0,0 +1,14 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Bill Thiede +Gary Burd +Jan Mercl <0xjnml@gmail.com> +Nick Owens +Tamás Gulácsi +Aaron Bieber diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/LICENSE b/Godeps/_workspace/src/github.com/cznic/fileutil/LICENSE new file mode 100644 index 00000000000..50bbdd2410f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The fileutil Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/Makefile b/Godeps/_workspace/src/github.com/cznic/fileutil/Makefile new file mode 100644 index 00000000000..5849cc20a2f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/Makefile @@ -0,0 +1,27 @@ +# Copyright (c) 2014 The fileutil authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +.PHONY: all clean editor todo + +all: editor + go vet + golint . + go install + make todo + +editor: + go fmt + go test -i + go test + go build + +todo: + @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go || true + @grep -n TODO *.go || true + @grep -n BUG *.go || true + @grep -n println *.go || true + +clean: + @go clean + rm -f y.output diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/README b/Godeps/_workspace/src/github.com/cznic/fileutil/README new file mode 100644 index 00000000000..f43d5f0047d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/README @@ -0,0 +1,16 @@ +This is a goinstall-able mirror of modified code already published at: +http://git.nic.cz/redmine/projects/gofileutil/repository + +Packages in this repository: + +Install: $go get github.com/cznic/fileutil +Godocs: http://godoc.org/github.com/cznic/fileutil + +Install: $go get github.com/cznic/fileutil/storage +Godocs: http://godoc.org/github.com/cznic/fileutil/storage + +Install: $go get github.com/cznic/fileutil/falloc +Godocs: http://godoc.org/github.com/cznic/fileutil/falloc + +Install: $go get github.com/cznic/fileutil/hdb +Godocs: http://godoc.org/github.com/cznic/fileutil/hdb diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/all_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/all_test.go new file mode 100644 index 00000000000..21a8fa948a8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/all_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTempFile(t *testing.T) { + f, err := TempFile("", "abc", "mno.xyz") + if err != nil { + t.Fatal(err) + } + + n := f.Name() + t.Log(n) + defer func() { + f.Close() + os.Remove(n) + }() + + base := filepath.Base(n) + if base == "abcmno.xyz" { + t.Fatal(base) + } + + if !strings.HasPrefix(base, "abc") { + t.Fatal(base) + } + + if !strings.HasSuffix(base, "mno.xyz") { + t.Fatal(base) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/LICENSE b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/LICENSE new file mode 100644 index 00000000000..1e92e33dd70 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of CZ.NIC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/README b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/README new file mode 100644 index 00000000000..23313fc5056 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/README @@ -0,0 +1,5 @@ +This is a goinstall-able mirror of modified code already published at: +https://git.nic.cz/redmine/projects/gofileutil/repository/show/falloc + +Install: $go get github.com/cznic/fileutil/falloc +Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/falloc diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/all_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/all_test.go new file mode 100644 index 00000000000..d70dcaff014 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/all_test.go @@ -0,0 +1,3105 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package falloc + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io/ioutil" + "log" + "math" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/cznic/fileutil" + "github.com/cznic/fileutil/storage" + "github.com/cznic/mathutil" +) + +var ( + blockFlag = flag.Uint("block", 256, "block size for some of the dev tests") + cacheTotalFlag = flag.Int64("cachemax", 1<<25, "cache total bytes") + cachedFlag = flag.Bool("cache", false, "enable caching store") + devFlag = flag.Bool("dev", false, "enable dev tests") + dropFlag = flag.Bool("drop", false, "drop system file cache for some of the dev tests before measurement") + fadviseFlag = flag.Bool("fadvise", false, "hint kernel about random file access") + nFlag = flag.Int("n", 1, "parameter for some of the dev tests") + probeFlag = flag.Bool("probe", false, "report store probe statistics") + optGo = flag.Int("go", 3, "GOMAXPROCS") +) + +func init() { + flag.Parse() + runtime.GOMAXPROCS(*optGo) +} + +func temp() (dir, name string) { + dir, err := ioutil.TempDir("", "test-falloc-") + if err != nil { + panic(err) + } + + name = filepath.Join(dir, "test.db") + return dir, name +} + +type balancedAcid struct { + storage.Accessor + nesting int +} + +func newBalancedAcid(store storage.Accessor) storage.Accessor { + return &balancedAcid{store, 0} +} + +func (b *balancedAcid) BeginUpdate() error { + if b.nesting < 0 { + return errors.New("BeginUpdate with nesting < 0") + } + + b.nesting++ + return nil +} + +func (b *balancedAcid) EndUpdate() error { + if b.nesting <= 0 { + return errors.New("EndUpdate with nesting <= 0") + } + + b.nesting-- + return nil +} + +func (b *balancedAcid) Close() error { + if b.nesting != 1 { + return fmt.Errorf("before Close(): nesting %d %p", b.nesting, b) + } + + if err := b.Accessor.Close(); err != nil { + return err + } + + if b.nesting != 1 { + return fmt.Errorf("after Close(): nesting %d", b.nesting) + } + + return nil +} + +func fopen(fn string) (f *File, err error) { + var store storage.Accessor + if store, err = storage.OpenFile(fn, os.O_RDWR, 0666); err != nil { + return + } + + var advise func(int64, int, bool) + if *fadviseFlag { + file := store.(*storage.FileAccessor).File + if err = fileutil.Fadvise(file, 0, 0, fileutil.POSIX_FADV_RANDOM); err != nil { + return + } + advise = func(off int64, len int, write bool) { + if err = fileutil.Fadvise(file, off, off+int64(len), fileutil.POSIX_FADV_DONTNEED); err != nil { + log.Fatal("advisor advise err", err) + } + } + } + + var prob *storage.Probe + if *probeFlag { + prob = storage.NewProbe(store, nil) + store = prob + } + if *cachedFlag { + if store, err = storage.NewCache(store, *cacheTotalFlag, advise); err != nil { + return + } + + if *probeFlag { + store = storage.NewProbe(store, prob) + } + } + f, err = Open(newBalancedAcid(store)) + return +} + +func fcreate(fn string) (f *File, err error) { + var store storage.Accessor + if store, err = storage.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666); err != nil { + return + } + + var advise func(int64, int, bool) + if *fadviseFlag { + file := store.(*storage.FileAccessor).File + if err = fileutil.Fadvise(file, 0, 0, fileutil.POSIX_FADV_RANDOM); err != nil { + return + } + advise = func(off int64, len int, write bool) { + if err = fileutil.Fadvise(file, off, off+int64(len), fileutil.POSIX_FADV_DONTNEED); err != nil { + log.Fatal("advisor advise err", err) + } + } + } + + var prob *storage.Probe + if *probeFlag { + prob = storage.NewProbe(store, nil) + store = prob + } + if *cachedFlag { + if store, err = storage.NewCache(store, *cacheTotalFlag, advise); err != nil { + return + } + + if *probeFlag { + store = storage.NewProbe(store, prob) + } + } + f, err = New(newBalancedAcid(store)) + return +} + +func probed(t *testing.T, f *File) { + if f == nil { + return + } + + dump := func(p *storage.Probe) { + t.Logf("OpsRd %d OpsWr %d BytesRd %d(avg %.1f) BytesWr %d(avg %.1f) SectorsRd %d(%d, +%d, x%.2f) SectorsWr %d(%d, +%d, x%.2f)", + p.OpsRd, p.OpsWr, + p.BytesRd, float64(p.BytesRd)/float64(p.OpsRd), + p.BytesWr, float64(p.BytesWr)/float64(p.OpsWr), + p.SectorsRd, + p.SectorsRd<<9, + p.SectorsRd<<9-p.BytesRd, + float64(p.SectorsRd<<9)/float64(p.BytesRd), + p.SectorsWr, + p.SectorsWr<<9, + p.SectorsWr<<9-p.BytesWr, + float64(p.SectorsWr<<9)/float64(p.BytesWr), + ) + } + + if ph, ok := f.Accessor().(*storage.Probe); ok { + dump(ph) + if c, ok := ph.Accessor.(*storage.Cache); ok { + if pl, ok := c.Accessor().(*storage.Probe); ok { + dump(pl) + } + } + } +} + +func (f *File) audit() (usedblocks, totalblocks int64, err error) { + defer func() { + if e := recover(); e != nil { + err = e.(error) + } + }() + + fi, err := f.f.Stat() + if err != nil { + panic(err) + } + + freemap := map[int64]int64{} + fp := int64(0) + buf := make([]byte, 22) + freeblocks := int64(0) + + // linear scan + for fp < fi.Size() { + totalblocks++ + typ, size := f.getInfo(fp >> 4) + f.read(buf[:1], fp+size<<4-1) + last := buf[0] + switch { + default: + panic("internal error") + case typ == 0: + if last != 0 { + panic(fmt.Errorf("@%#x used empty, last @%#x: %#x != 0", fp, fp+size<<4-1, last)) + } + case typ >= 0x1 && typ <= 0xed: + if last >= 0xfe { + panic(fmt.Errorf("@%#x used short, last @%#x: %#x > 0xfe", fp, fp+size<<4-1, last)) + } + case typ >= 0xee && typ <= 0xfb: + if last > 1 { + panic(fmt.Errorf("@%#x used esc short, last @%#x: %#x > 1", fp, fp+size<<4-1, last)) + } + case typ == 0xfc: + f.read(buf[:2], fp+1) + switch n := int(buf[0])<<8 + int(buf[1]); { + default: + panic(fmt.Errorf("@%#x used long, illegal content length %#x < 0xee(238)", fp, n)) + case n >= 0xee && n <= 0xf0f0: + if last >= 0xfe { + panic(fmt.Errorf("@%#x used long, last @%#x: %#x > 0xfe", fp, fp+size<<4-1, last)) + } + case n >= 0xf0f1 && n <= 0xffff: + if last > 1 { + panic(fmt.Errorf("@%#x used esc long, last @%#x: %#x > 1", fp, fp+size<<4-1, last)) + } + } + case typ == 0xfd: + if last != 0 { + panic(fmt.Errorf("@%#x reloc, last @%#x: %#x != 0", fp, fp+size<<4-1, last)) + } + + var target int64 + f.read(buf[:7], fp+1) + (*Handle)(&target).Get(buf) + if target >= f.atoms { + panic(fmt.Errorf("@%#x illegal reloc, target %#x > f.atoms(%#x)", fp, target, f.atoms)) + } + + ttyp, _ := f.getInfo(target) + if ttyp >= 0xfe { + panic(fmt.Errorf("@%#x reloc, points to unused @%#x", fp, target)) + } + + if ttyp == 0xfd { + panic(fmt.Errorf("@%#x reloc, points to reloc @%#x", fp, target)) + } + case typ == 0xfe: + if size < 2 { + panic(fmt.Errorf("@%#x illegal free block, atoms %d < 2", fp, size)) + } + + if fp>>4 < f.canfree { + panic(fmt.Errorf("@%#x illegal free block @ < f.canfree", fp)) + } + + f.read(buf[:22], fp) + var prev, next, sz int64 + (*Handle)(&prev).Get(buf[1:]) + (*Handle)(&next).Get(buf[8:]) + f.checkPrevNext(fp, prev, next) + f.read(buf[:7], fp+size<<4-8) + (*Handle)(&sz).Get(buf) + if sz != size { + panic(fmt.Errorf("@%#x mismatch size, %d != %d", fp, sz, size)) + } + + if last != 0xfe { + panic(fmt.Errorf("@%#x free atom, last @%#x: %#x != 0xff", fp, fp+size<<4-1, last)) + } + freemap[fp>>4] = size + freeblocks++ + case typ == 0xff: + f.read(buf[:14], fp+1) + var prev, next int64 + (*Handle)(&prev).Get(buf) + (*Handle)(&next).Get(buf[7:]) + f.checkPrevNext(fp, prev, next) + if last != 0xff { + panic(fmt.Errorf("@%#x free atom, last @%#x: %#x != 0xff", fp, fp+size<<4-1, last)) + } + freemap[fp>>4] = size + freeblocks++ + } + fp += size << 4 + } + usedblocks = totalblocks - freeblocks + + // check free table + for size := len(f.freetab) - 1; size > 0; size-- { + var prev, next, fprev int64 + this := f.freetab[size] + for this != 0 { + sz, ok := freemap[this] + if !ok { + panic(fmt.Errorf("bad freetab[%d] item @%#x", size, this)) + } + + delete(freemap, this) + + if sz < int64(size) { + panic(fmt.Errorf("bad freetab[%d] item size @%#x %d", size, this, sz)) + } + + if sz == 1 { + f.read(buf[:15], this<<4) + (*Handle)(&fprev).Get(buf[1:]) + if fprev != prev { + panic(fmt.Errorf("bad fprev %#x, exp %#x", fprev, prev)) + } + + (*Handle)(&next).Get(buf[8:]) + } else { + f.read(buf, this<<4) + (*Handle)(&fprev).Get(buf[1:]) + if fprev != prev { + panic(fmt.Errorf("bad fprev %#x, exp %#x", fprev, prev)) + } + var fsz int64 + (*Handle)(&fsz).Get(buf[15:]) + if fsz != sz { + panic(fmt.Errorf("bad fsz %d @%#x, exp %#x", fsz, this<<4, sz)) + } + + (*Handle)(&next).Get(buf[8:]) + } + + prev, this = this, next + } + } + + if n := len(freemap); n != 0 { + for h, s := range freemap { + panic(fmt.Errorf("%d lost free blocks in freemap, e.g. %d free atoms @%#x", n, s, h)) + } + } + + return + +} + +func (f *File) checkPrevNext(fp, prev, next int64) { + if prev != 0 && prev < f.canfree { + panic(fmt.Errorf("@%#x illegal free atom, prev %#x < f.canfree(%#x)", fp, prev, f.canfree)) + } + + if prev >= f.atoms { + panic(fmt.Errorf("@%#x illegal free atom, prev %#x > f.atoms", fp, prev)) + } + + if next != 0 && next < f.canfree { + panic(fmt.Errorf("@%#x illegal free atom, next %#x < f.canfree(%#x)", fp, next, f.canfree)) + } + + if next >= f.atoms { + panic(fmt.Errorf("@%#x illegal free atom, next %#x > f.atoms", fp, next)) + } +} + +func reaudit(t *testing.T, f *File, fn string) (of *File) { + var err error + if _, _, err := f.audit(); err != nil { + t.Fatal(err) + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if of, err = fopen(fn); err != nil { + t.Fatal(err) + } + + if _, _, err := of.audit(); err != nil { + t.Fatal(err) + } + + return +} + +func TestCreate(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := os.Remove(name) + if err != nil { + t.Fatal(err) + } + }() + + f.Accessor().Sync() + probed(t, f) + if err = f.Close(); err != nil { + t.Log(f.f.(*balancedAcid).nesting) + t.Fatal(err) + } + + b, err := ioutil.ReadFile(name) + if err != nil { + t.Fatal(err) + } + + x := b[:16] + if !bytes.Equal(x, hdr) { + t.Fatalf("\n% x\n% x", x, hdr) + } + + x = b[16:32] + if !bytes.Equal(x, empty) { + t.Fatalf("\n% x\n% x", x, hdr) + } +} + +func TestOpen(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + probed(t, f) + ec := f.Close() + er := os.Remove(name) + if ec != nil { + t.Fatal(ec) + } + + if er != nil { + t.Fatal(er) + } + }() + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + for i, p := range f.freetab { + if p != 0 { + t.Fatal(i+1, p) + } + } +} + +func alloc(f *File, b []byte) (y int64) { + if h, err := f.Alloc(b); err != nil { + panic(err) + } else { + y = int64(h) + } + return +} + +func realloc(f *File, atom int64, b []byte, keepHandle bool) (y int64) { + if h, err := f.Realloc(Handle(atom), b, keepHandle); err != nil { + panic(err) + } else { + y = int64(h) + } + return +} + +func testContentEncodingDecoding(t *testing.T, min, max int) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + ec := f.Close() + er := os.Remove(name) + if ec != nil { + t.Fatal(ec) + } + + if er != nil { + t.Fatal(er) + } + }() + + b := make([]byte, max) + r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + t.Fatal(err) + } + + blocks := int64(3) + a := make([]int64, 0, 4*(max-min+1)) + for cl := min; cl <= max; cl++ { + src := b[:cl] + for i := range src { + b[i] = byte(r.Next()) + } + a = append(a, alloc(f, src)) + blocks++ + if cl == 0 { + continue + } + + for i := range src { + b[i] = byte(r.Next()) + } + src[cl-1] = 0xfd + a = append(a, alloc(f, src)) + blocks++ + for i := range src { + b[i] = byte(r.Next()) + } + src[cl-1] = 0xfe + a = append(a, alloc(f, src)) + blocks++ + for i := range src { + b[i] = byte(r.Next()) + } + src[cl-1] = 0xff + a = append(a, alloc(f, src)) + blocks++ + } + + f.Accessor().Sync() + probed(t, f) + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + r.Seek(0) + ai := 0 + for cl := min; cl <= max; cl++ { + h := a[ai] + ai++ + src := b[:cl] + for i := range src { + b[i] = byte(r.Next()) + } + got, _ := f.readUsed(h) + if !bytes.Equal(src, got) { + t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got) + } + if cl == 0 { + continue + } + + for i := range src { + b[i] = byte(r.Next()) + } + src[cl-1] = 0xfd + h = a[ai] + ai++ + got, _ = f.readUsed(h) + if !bytes.Equal(src, got) { + t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got) + } + + for i := range src { + b[i] = byte(r.Next()) + } + src[cl-1] = 0xfe + h = a[ai] + ai++ + got, _ = f.readUsed(h) + if !bytes.Equal(src, got) { + t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got) + } + + for i := range src { + b[i] = byte(r.Next()) + } + src[cl-1] = 0xff + h = a[ai] + ai++ + got, _ = f.readUsed(h) + if !bytes.Equal(src, got) { + t.Fatalf("cl %d atom %#x\nexp % x\ngot % x", cl, h, src, got) + } + } + + auditblocks, _, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if auditblocks != blocks { + t.Fatal(auditblocks, blocks) + } + + if f = reaudit(t, f, name); err != nil { + t.Fatal(err) + } +} + +func TestContentEncodingDecoding(t *testing.T) { + testContentEncodingDecoding(t, 0, 1024) + testContentEncodingDecoding(t, 61680-17, 61680) +} + +type freeItem struct { + size int64 + head int64 +} + +func (f *File) reportFree() (report []freeItem) { + for size, head := range f.freetab { + if size != 0 && head != 0 { + report = append(report, freeItem{int64(size), head}) + } + } + return +} + +func free(f *File, h int64) { + if err := f.Free(Handle(h)); err != nil { + panic(err) + } +} + +func testFreeTail(t *testing.T, b []byte) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + ec := f.Close() + er := os.Remove(name) + if ec != nil { + t.Fatal(ec) + } + + if er != nil { + t.Fatal(er) + } + }() + + fs0 := f.atoms + used0, total0, err := f.audit() + if err != nil { + panic(err) + } + + if used0 != total0 { + t.Fatal(used0, total0) + } + + handle := alloc(f, b) + free(f, handle) + if fs1 := f.atoms; fs1 != fs0 { + t.Fatal(fs1, fs0) + } + + if rep := f.reportFree(); len(rep) != 0 { + t.Fatal(rep) + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + used, total, err := f.audit() + if err != nil { + panic(err) + } + + if used != used0 { + t.Fatal(used, used0) + } + + if total != total0 { + t.Fatal(total, total0) + } +} + +func TestFreeTail(t *testing.T) { + b := make([]byte, 61680) + for n := 0; n <= 253+16; n++ { + data := b[:n] + testFreeTail(t, data) + if n == 0 { + continue + } + + data[n-1] = 0xff + testFreeTail(t, data) + data[n-1] = 0 + } + + for n := 61680 - 16; n <= 61680; n++ { + data := b[:n] + testFreeTail(t, data) + data[n-1] = 0xff + testFreeTail(t, data) + data[n-1] = 0 + } +} + +func testFreeTail2(t *testing.T, b []byte) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + ec := f.Close() + er := os.Remove(name) + if ec != nil { + t.Fatal(ec) + } + + if er != nil { + t.Fatal(er) + } + }() + + fs0 := f.atoms + used0, total0, err := f.audit() + if err != nil { + panic(err) + } + + if used0 != total0 { + t.Fatal(used0, total0) + } + + handle := alloc(f, b) + handle2 := alloc(f, b) + free(f, handle) + free(f, handle2) + if fs1 := f.atoms; fs1 != fs0 { + t.Fatal(fs1, fs0) + } + + if rep := f.reportFree(); len(rep) != 0 { + t.Fatal(rep) + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + used, total, err := f.audit() + if err != nil { + panic(err) + } + + if used != used0 { + t.Fatal(used, used0) + } + + if total != total0 { + t.Fatal(total, total0) + } +} + +func TestFreeTail2(t *testing.T) { + b := make([]byte, 61680) + for n := 0; n <= 253+16; n++ { + data := b[:n] + testFreeTail2(t, data) + if n == 0 { + continue + } + + data[n-1] = 0xff + testFreeTail2(t, data) + data[n-1] = 0 + } + + for n := 61680 - 16; n <= 61680; n++ { + data := b[:n] + testFreeTail2(t, data) + data[n-1] = 0xff + testFreeTail2(t, data) + data[n-1] = 0 + } +} + +func testFreeIsolated(t *testing.T, b []byte) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + ec := f.Close() + er := os.Remove(name) + if ec != nil { + t.Fatal(ec) + } + + if er != nil { + t.Fatal(er) + } + }() + + rqAtoms := rq2Atoms(len(b)) + left := alloc(f, nil) + handle := alloc(f, b) + right := alloc(f, nil) + + fs0 := f.atoms + used0, total0, err := f.audit() + if err != nil { + panic(err) + } + + if used0 != total0 { + t.Fatal(used0, total0) + } + + free(f, handle) + if fs1 := f.atoms; fs1 != fs0 { + t.Fatal(fs1, fs0) + } + + rep := f.reportFree() + if len(rep) != 1 { + t.Fatal(rep) + } + + if x := rep[0]; x.size != rqAtoms || x.head != handle { + t.Fatal(x) + } + + used, total, err := f.audit() + if err != nil { + panic(err) + } + + if n, free := f.getSize(left); n != 1 || free { + t.Fatal(n, free) + } + + if n, free := f.getSize(right); n != 1 || free { + t.Fatal(n, free) + } + + if used != used0-1 { + t.Fatal(used, used0) + } + + if total != total0 { + t.Fatal(total, total0) + } + + if free := total - used; free != 1 { + t.Fatal(free) + } + + // verify persisted file correct + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + if fs1 := f.atoms; fs1 != fs0 { + t.Fatal(fs1, fs0) + } + + rep = f.reportFree() + if len(rep) != 1 { + t.Fatal(rep) + } + + if x := rep[0]; x.size != rqAtoms || x.head != handle { + t.Fatal(x) + } + + used, total, err = f.audit() + if err != nil { + panic(err) + } + + if n, free := f.getSize(left); n != 1 || free { + t.Fatal(n, free) + } + + if n, free := f.getSize(right); n != 1 || free { + t.Fatal(n, free) + } + + if used != used0-1 { + t.Fatal(used, used0) + } + + if total != total0 { + t.Fatal(total, total0) + } + + if free := total - used; free != 1 { + t.Fatal(free) + } + +} + +func TestFreeIsolated(t *testing.T) { + b := make([]byte, 61680) + for n := 0; n <= 253+16; n++ { + data := b[:n] + testFreeIsolated(t, data) + } + + for n := 61680 - 16; n <= 61680; n++ { + data := b[:n] + testFreeIsolated(t, data) + } +} + +func testFreeBlockList(t *testing.T, a, b int) { + var h [2]int64 + + t.Log(a, b) + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + alloc(f, nil) + h[0] = alloc(f, nil) + alloc(f, nil) + h[1] = alloc(f, nil) + alloc(f, nil) + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if used-used0 != 5 || total-total0 != 5 || used != total { + t.Fatal(used0, total0, used, total) + } + + free(f, h[a]) + free(f, h[b]) + + used, total, err = f.audit() + if err != nil { + t.Fatal(err) + } + + if used-used0 != 3 || total-total0 != 5 || total-used != 2 { + t.Fatal(used0, total0, used, total) + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + used, total, err = f.audit() + if err != nil { + t.Fatal(err) + } + + if used-used0 != 3 || total-total0 != 5 || total-used != 2 { + t.Fatal(used0, total0, used, total) + } +} + +func TestFreeBlockList(t *testing.T) { + testFreeBlockList(t, 0, 1) + testFreeBlockList(t, 1, 0) +} + +func testFreeBlockList2(t *testing.T, a, b, c int) { + var h [3]int64 + + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + alloc(f, nil) + h[0] = alloc(f, nil) + alloc(f, nil) + h[1] = alloc(f, nil) + alloc(f, nil) + h[2] = alloc(f, nil) + alloc(f, nil) + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if used-used0 != 7 || total-total0 != 7 || used != total { + t.Fatal(used0, total0, used, total) + } + + free(f, h[a]) + free(f, h[b]) + free(f, h[c]) + + used, total, err = f.audit() + if err != nil { + t.Fatal(err) + } + + if used-used0 != 4 || total-total0 != 7 || total-used != 3 { + t.Fatal(used0, total0, used, total) + } + + if err := f.Close(); err != nil { + t.Fatal(err) + } + + f = nil + runtime.GC() + if f, err = fopen(name); err != nil { + t.Fatal(err) + } + + used, total, err = f.audit() + if err != nil { + t.Fatal(err) + } + + if used-used0 != 4 || total-total0 != 7 || total-used != 3 { + t.Fatal(used0, total0, used, total) + } +} + +func TestFreeBlockList2(t *testing.T) { + testFreeBlockList2(t, 0, 1, 2) + testFreeBlockList2(t, 0, 2, 1) + testFreeBlockList2(t, 1, 0, 2) + testFreeBlockList2(t, 1, 2, 0) + testFreeBlockList2(t, 2, 0, 1) + testFreeBlockList2(t, 2, 1, 0) +} + +var crng *mathutil.FC32 + +func init() { + var err error + if crng, err = mathutil.NewFC32(0, math.MaxInt32, true); err != nil { + panic(err) + } +} + +func content(b []byte, h int64) (c []byte) { + crng.Seed(h) + crng.Seek(0) + c = b[:crng.Next()%61681] + for i := range c { + c[i] = byte(crng.Next()) + } + return +} + +func testFreeBlockList3(t *testing.T, n, mod int) { + rng, err := mathutil.NewFC32(0, n-1, true) + if err != nil { + t.Fatal(err) + } + + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + ha := make([]int64, n) + b := make([]byte, 61680) + for i := range ha { + h := f.atoms + ha[i] = h + c := content(b, h) + if alloc(f, c) != h { + t.Fatal(h) + } + } + f = reaudit(t, f, name) + del := map[int64]bool{} + for _ = range ha { + i := rng.Next() + if i%mod != 0 { + h := ha[i] + free(f, h) + del[h] = true + } + } + f = reaudit(t, f, name) + for _, h := range ha { + if !del[h] { + exp := content(b, h) + got, _ := f.readUsed(h) + if !bytes.Equal(exp, got) { + t.Fatal(len(got), len(exp)) + } + } + } +} + +func TestFreeBlockList3(t *testing.T) { + testFreeBlockList3(t, 111, 1) + testFreeBlockList3(t, 151, 2) + testFreeBlockList3(t, 170, 3) + testFreeBlockList3(t, 170, 4) +} + +func TestRealloc1(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + exp := c[:15] + if handle := realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc1Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + exp := c[:15] + if handle := realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc2(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, c[:31]) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + exp := c[:15] + if handle := realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc2Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, c[:31]) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + exp := c[:15] + if handle := realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc3(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + exp := c[:31] + var handle int64 + if handle = realloc(f, h10, exp, false); handle == h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc3Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + exp := c[:31] + var handle int64 + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 1 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc4Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, c[:31]) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + exp := c[:47] + var handle int64 + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 2 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc5(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h15 := alloc(f, nil) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + free(f, h15) + exp := c[:31] + var handle int64 + if handle = realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc5Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h15 := alloc(f, nil) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + free(f, h15) + exp := c[:31] + var handle int64 + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc6(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h15 := alloc(f, c[:31]) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + free(f, h15) + exp := c[:31] + var handle int64 + if handle = realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != 0 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRealloc6Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + h10 := alloc(f, nil) + h15 := alloc(f, c[:31]) + h20 := alloc(f, nil) + + used0, total0, err := f.audit() + if err != nil { + t.Fatal(err) + } + + free(f, h15) + exp := c[:31] + var handle int64 + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(handle); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != 0 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc1(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + used0, total0, err := f.audit() // c+3, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:15] + if handle = realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() // c+2, c+2 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc1Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + used0, total0, err := f.audit() // c+3, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:15] + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + if got, _ := f.readUsed(h20); len(got) != 0 { + t.Fatal(len(got), 0) + } + + used, total, err := f.audit() // c+2, c+2 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -1 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc2(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + free(f, h20) + + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:31] + if handle = realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+1, c+1 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc2Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + h20 := alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + free(f, h20) + + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:31] + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+1, c+1 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc3(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + h20 := alloc(f, b[:31]) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + free(f, h20) + + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:31] + if handle = realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+1, c+1 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc3Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + h20 := alloc(f, b[:31]) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + free(f, h20) + + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:31] + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+1, c+1 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != -2 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc4(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + _ = alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:47], true); handle != h10 { + t.Fatal(handle, h10) + } + + _ = alloc(f, nil) + + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + used0, total0, err := f.audit() // c+4, c+5 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:47] + if handle = realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+4, c+4 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != -1 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc4Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + _ = alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:47], true); handle != h10 { + t.Fatal(handle, h10) + } + + _ = alloc(f, nil) + + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + used0, total0, err := f.audit() // c+4, c+5 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:47] + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+4, c+4 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != -1 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc5(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + _ = alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + _ = alloc(f, nil) + + used0, total0, err := f.audit() // c+4, c+4 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:47] + if handle = realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+4, c+5 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc5Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, nil) + _ = alloc(f, nil) + var handle int64 + if handle = realloc(f, h10, b[:31], true); handle != h10 { + t.Fatal(handle, h10) + } + + _ = alloc(f, nil) + + used0, total0, err := f.audit() // c+4, c+4 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:47] + if handle = realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+4, c+5 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 1 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc6(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, b[:31]) + h20 := alloc(f, nil) + _ = alloc(f, nil) + free(f, h20) + + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:15] + if handle := realloc(f, h10, exp, false); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestRelocRealloc6Keep(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + + h10 := alloc(f, b[:31]) + h20 := alloc(f, nil) + _ = alloc(f, nil) + free(f, h20) + + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + c := content(b, 10) + exp := c[:15] + if handle := realloc(f, h10, exp, true); handle != h10 { + t.Fatal(handle, h10) + } + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, exp) { + t.Fatal(len(got), len(exp)) + } + + used, total, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 0 || diftotal != 0 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestFreespaceReuse(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + c10 := c[0 : 0+15] + c20 := c[16:63] + c50 := c[64 : 64+15] + h10 := alloc(f, c10) + h201 := alloc(f, nil) + h202 := alloc(f, nil) + h203 := alloc(f, nil) + h50 := alloc(f, c50) + free(f, h201) + free(f, h202) + free(f, h203) + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + h20 := alloc(f, c20) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) { + t.Fatal() + } + + if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) { + t.Fatal() + } + + if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) { + t.Fatal() + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) { + t.Fatal() + } + + if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) { + t.Fatal() + } + + if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) { + t.Fatal() + } + + used, total, err := f.audit() // c+3, c+3 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 0 || free != 0 { + t.Fatal(difused, diftotal, free) + } +} + +func TestFreespaceReuse2(t *testing.T) { + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + c := content(b, 10) + + c10 := c[0 : 0+15] + c20 := c[16:47] + c50 := c[64 : 64+15] + h10 := alloc(f, c10) + h201 := alloc(f, nil) + h202 := alloc(f, nil) + h203 := alloc(f, nil) + h50 := alloc(f, c50) + free(f, h201) + free(f, h202) + free(f, h203) + used0, total0, err := f.audit() // c+2, c+3 + if err != nil { + t.Fatal(err) + } + + h20 := alloc(f, c20) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) { + t.Fatal() + } + + if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) { + t.Fatal() + } + + if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) { + t.Fatal() + } + + f = reaudit(t, f, name) + + if got, _ := f.readUsed(h10); !bytes.Equal(got, c10) { + t.Fatal() + } + + if got, _ := f.readUsed(h20); !bytes.Equal(got, c20) { + t.Fatal() + } + + if got, _ := f.readUsed(h50); !bytes.Equal(got, c50) { + t.Fatal() + } + + used, total, err := f.audit() // c+3, c+4 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != 1 || diftotal != 1 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func testBug1(t *testing.T, swap bool) { + // Free lists table item for size 3856 points to list of free blocks + // NOT of size 3856 but at least 3856. + + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + _ = alloc(f, nil) + b := make([]byte, 61680) + f1 := alloc(f, b) + f2 := alloc(f, b) + _ = alloc(f, nil) + + used0, total0, err := f.audit() // c+4, c+4 + if err != nil { + t.Fatal(err) + } + + if swap { + f1, f2 = f2, f1 + } + free(f, f1) + free(f, f2) + _ = alloc(f, nil) + + f = reaudit(t, f, name) + + used, total, err := f.audit() // c+3, c+4 + if err != nil { + t.Fatal(err) + } + + if difused, diftotal, free := used-used0, total-total0, total-used; difused != -1 || diftotal != 0 || free != 1 { + t.Fatal(difused, diftotal, free) + } +} + +func TestBug1(t *testing.T) { + testBug1(t, false) + testBug1(t, true) +} + +func TestMix(t *testing.T) { + if testing.Short() { + t.Log("skipped") + return + } + + const ( + n = 1 << 10 + ) + + if testing.Short() { + t.Log("skipped") + return + } + + t.Log(n) + dir, name := temp() + defer os.RemoveAll(dir) + + f, err := fcreate(name) + if err != nil { + t.Fatal(err) + } + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + f = nil + runtime.GC() + os.Remove(name) + }() + + b := make([]byte, 61680) + rng, err := mathutil.NewFC32(0, n-1, true) + if err != nil { + t.Fatal(err) + } + + ha := make([]int64, n) + payload := 0 + + t0 := time.Now() + // Alloc n block with upper half of content + for _ = range ha { + r := rng.Next() + c := content(b, int64(r)) + c = c[len(c)/2:] + ha[r] = alloc(f, c) + payload += len(c) + } + dt := float64(time.Now().Sub(t0)) / 1e9 + t.Logf("write time A %.3g", dt) + + // verify + f = reaudit(t, f, name) + t.Logf("size A %d for %d bytes (fill factor %3.1f%%)", f.atoms<<4, payload, 100*float64(payload)/float64(f.atoms<<4)) + t0 = time.Now() + for _ = range ha { + r := rng.Next() + c := content(b, int64(r)) + c = c[len(c)/2:] + if got, _ := f.readUsed(ha[r]); !bytes.Equal(got, c) { + t.Fatal() + } + } + dt = float64(time.Now().Sub(t0)) / 1e9 + t.Logf("read time A %.3g", dt) + // free half of the blocks + t0 = time.Now() + for i := 0; i < n/2; i++ { + free(f, ha[i]) + ha[i] = 0 + } + dt = float64(time.Now().Sub(t0)) / 1e9 + t.Logf("free time A %.3g", dt) + + // verify + f = reaudit(t, f, name) + t.Logf("size B %d (freeing half of the blocks)", f.atoms<<4) + t0 = time.Now() + for _ = range ha { + r := rng.Next() + h := ha[r] + if h == 0 { + continue + } + + c := content(b, int64(r)) + c = c[len(c)/2:] + if got, _ := f.readUsed(h); !bytes.Equal(got, c) { + t.Fatal() + } + } + dt = float64(time.Now().Sub(t0)) / 1e9 + t.Logf("read time B %.3g", dt) + + // reloc extend + t0 = time.Now() + for _ = range ha { + r := rng.Next() + h := ha[r] + if h == 0 { + continue + } + + c := content(b, int64(r)) + //f = reaudit(t, f, name) + if h2 := realloc(f, h, c, true); h2 != h { + t.Fatal() + } + } + dt = float64(time.Now().Sub(t0)) / 1e9 + t.Logf("realoc time B %.3g", dt) + + // verify + f = reaudit(t, f, name) + t.Logf("size C %d for %d bytes (reallocated all used blocks to double size, fill factor %3.1f%%", f.atoms<<4, payload, 100*float64(payload)/float64(f.atoms<<4)) + + t0 = time.Now() + for _ = range ha { + r := rng.Next() + h := ha[r] + if h == 0 { + continue + } + + c := content(b, int64(r)) + if got, _ := f.readUsed(ha[r]); !bytes.Equal(got, c) { + t.Fatal() + } + } + dt = float64(time.Now().Sub(t0)) / 1e9 + t.Logf("read time C %.3g", dt) +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/docs.go b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/docs.go new file mode 100644 index 00000000000..21772fcd06e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/docs.go @@ -0,0 +1,251 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +/* + +WIP: Package falloc provides allocation/deallocation of space within a +file/store (WIP, unstable API). + +Overall structure: + File == n blocks. + Block == n atoms. + Atom == 16 bytes. + +x6..x0 == least significant 7 bytes of a 64 bit integer, highest (7th) byte is +0 and is not stored in the file. + +Block first byte + +Aka block type tag. + +------------------------------------------------------------------------------ + +0xFF: Free atom (free block of size 1). + +------++---------++---------++------+ + | 0 || 1...7 || 8...14 || 15 | + +------++---------++---------++------+ + | 0xFF || p6...p0 || n6...n0 || 0xFF | + +------++---------++---------++------+ + +Link to the previous free block (atom addressed) is p6...p0, next dtto in +n6...n0. Doubly linked lists of "compatible" free blocks allows for free space +reclaiming and merging. "Compatible" == of size at least some K. Heads of all +such lists are organized per K or intervals of Ks elsewhere. + +------------------------------------------------------------------------------ + +0xFE: Free block, size == s6...s0 atoms. + +------++---------++---------++---------++-- + | +0 || 1...7 || 8...14 || 15...21 || 22...16*size-1 + +------++---------++---------++---------++-- + | 0xFE || p6...p0 || n6...n0 || s6...s0 || ... + +------++---------++---------++---------++-- + +Prev and next links as in the 0xFF first byte case. End of this block - see +"Block last byte": 0xFE bellow. Data between == undefined. + +------------------------------------------------------------------------------ + +0xFD: Relocated block. + +------++---------++-----------++------+ + | 0 || 1...7 || 8...14 || 15 | + +------++---------++-----------++------+ + | 0xFD || r6...r0 || undefined || 0x00 | // == used block + +------++---------++-----------++------+ + +Relocation link is r6..r0 == atom address. Relocations MUST NOT chain and MUST +point to a "content" block, i.e. one with the first byte in 0x00...0xFC. + +Relocated block allows to permanently assign a handle/file pointer ("atom" +address) to some content and resize the content anytime afterwards w/o having +to update all the possible existing references to the original handle. + +------------------------------------------------------------------------------ + +0xFC: Used long block. + +------++---------++--------------------++---------+---+ + | 0 || 1...2 || 3...N+2 || | | + +------++---------++--------------------++---------+---+ + | 0xFC || n1...n0 || N bytes of content || padding | Z | + +------++---------++--------------------++---------+---+ + +This block type is used for content of length in N == 238...61680 bytes. N is +encoded as a 2 byte unsigned integer n1..n0 in network byte order. Values +bellow 238 are reserved, those content lengths are to be carried by the +0x00..0xFB block types. + + 1. n in 0x00EE...0xF0F0 is used for content under the same rules + as in the 0x01..0xED type. + + 2. If the last byte of the content is not the last byte of an atom then + the last byte of the block is 0x00. + + 3. If the last byte of the content IS the last byte of an atom: + + 3.1 If the last byte of content is in 0x00..0xFD then everything is OK. + + 3.2 If the last byte of content is 0xFE or 0xFF then the escape + via n > 0xF0F0 MUST be used AND the block's last byte is 0x00 or 0x01, + meaning value 0xFE and 0xFF respectively. + + 4. n in 0xF0F1...0xFFFF is like the escaped 0xEE..0xFB block. + N == 13 + 16(n - 0xF0F1). + +Discussion of the padding and Z fields - see the 0x01..0xED block type. + +------------------------------------------------------------------------------ + +0xEE...0xFB: Used escaped short block. + +---++----------------------++---+ + | 0 || 1...N-1 || | + +---++----------------------++---+ + | X || N-1 bytes of content || Z | + +---++----------------------++---+ + +N == 15 + 16(X - 0xEE). Z is the content last byte encoded as follows. + +case Z == 0x00: The last byte of content is 0xFE + +case Z == 0x01: The last byte of content is 0xFF + +------------------------------------------------------------------------------ + +0x01...0xED: Used short block. + +---++--------------------++---------+---+ + | 0 || 1...N || | | + +---++--------------------++---------+---+ + | N || N bytes of content || padding | Z | + +---++--------------------++---------+---+ + +This block type is used for content of length in 1...237 bytes. The value of +the "padding" field, if of non zero length, is undefined. + +If the last byte of content is the last byte of an atom (== its file byte +offset & 0xF == 0xF) then such last byte MUST be in 0x00...0xFD. + +If the last byte of content is the last byte of an atom AND the last byte of +content is 0xFE or 0xFF then the short escape block type (0xEE...0xFB) MUST be +used. + +If the last byte of content is not the last byte of an atom, then the last byte +of such block, i.e. the Z field, which is also a last byte of some atom, MUST +be 0x00 (i.e. the used block marker). Other "tail" values are reserved. + +------------------------------------------------------------------------------ + +0x00: Used empty block. + +------++-----------++------+ + | 0 || 1...14 || 15 | + +------++-----------++------+ + | 0x00 || undefined || 0x00 | // == used block, other "tail" values reserved. + +------++-----------++------+ + +All of the rules for 0x01..0xED applies. Depicted only for its different +semantics (e.g. an allocated [existing] string but with length of zero). + +============================================================================== + +Block last byte + +------------------------------------------------------------------------------ + +0xFF: Free atom. Layout - see "Block first byte": FF. + +------------------------------------------------------------------------------ + +0xFE: Free block, size n atoms. Preceding 7 bytes == size (s6...s0) of the free +block in atoms, network byte order + --++---------++------+ + || -8...-2 || -1 | + --++---------++------+ + ... || s6...s0 || 0xFE | <- block's last byte + --++---------++------+ + +Layout at start of this block - see "Block first byte": FE. + +------------------------------------------------------------------------------ + +0x00...0xFD: Used (non free) block. + +============================================================================== + +Free lists table + +The free lists table content is stored in the standard layout of a used block. + +A table item is a 7 byte size field followed by a 7 byte atom address field +(both in network byte order), thus every item is 14 contiguous bytes. The +item's address field is pointing to a free block. The size field determines +the minimal size (in atoms) of free blocks on that list. + +The free list table is n above items, thus the content has 14n bytes. Note that +the largest block content is 61680 bytes and as there are 14 bytes per table +item, so the table is limited to at most 4405 entries. + +Items in the table do not have to be sorted according to their size field values. + +No two items can have the same value of the size field. + +When freeing blocks, the block MUST be linked into an item list with the +highest possible size field, which is less or equal to the number of atoms in +the new free block. + +When freeing a block, the block MUST be first merged with any adjacent free +blocks (thus possibly creating a bigger free block) using information derived +from the adjacent blocks first and last bytes. Such merged free blocks MUST be +removed from their original doubly linked lists. Afterwards the new bigger free +block is put to the free list table in the appropriate item. + +Items with address field == 0 are legal. Such item is a placeholder for a empty +list of free blocks of the item's size. + +Items with size field == 0 are legal. Such item is a placeholder, used e.g. to +avoid further reallocations/redirecting of the free lists table. + +The largest possible allocation request (for content length 61680 bytes) is +0xF10 (3856) atoms. All free blocks of this or bigger size are presumably put +into a single table item with the size 3856. It may be useful to additionally +have a free lists table item which links free blocks of some bigger size (say +1M+) and then use the OS sparse file support (if present) to save the physical +space used by such free blocks. + +Smaller (<3856 atoms) free blocks can be organized exactly (every distinct size +has its table item) or the sizes can run using other schema like e.g. "1, 2, +4, 8, ..." (powers of 2) or "1, 2, 3, 5, 8, 13, ..." (the Fibonacci sequence) +or they may be fine tuned to a specific usage pattern. + +============================================================================== + +Header + +The first block of a file (atom address == file offset == 0) is the file header. +The header block has the standard layout of a used short non escaped block. + +Special conditions apply: The header block and its content MUST be like this: + + +------+---------+---------+------+ + | 0 | 1...7 | 8...14 | 15 | + +------+---------+---------+------+ + | 0x0F | m6...m0 | f6...f0 | FLTT | + +------+---------+---------+------+ + +m6..m0 is a "magic" value 0xF1C1A1FE51B1E. + +f6...f0 is the atom address of the free lists table (discussed elsewhere). +If f6...f0 == 0x00 the there is no free lists table (yet). + +FLTT describes the type of the Free List Table. Currently defined values: + +------------------------------------------------------------------------------ + +FLTT == 0: Free List Table is fixed at atom address 2. It has a fixed size for 3856 entries +for free list of size 1..3855 atoms and the last is for the list of free block >= 3856 atoms. +*/ +package falloc + +const ( + INVALID_HANDLE = Handle(-1) +) diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/error.go b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/error.go new file mode 100644 index 00000000000..dad3d29e684 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/error.go @@ -0,0 +1,130 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package falloc + +import "fmt" + +// EBadRequest is an error produced for invalid operation, e.g. for data of more than maximum allowed. +type EBadRequest struct { + Name string + Size int +} + +func (e *EBadRequest) Error() string { + return fmt.Sprintf("%s: size %d", e.Name, e.Size) +} + +// EClose is a file/store close error. +type EClose struct { + Name string + Err error +} + +func (e *EClose) Error() string { + return fmt.Sprintf("%sx: %s", e.Name, e.Err) +} + +// ECorrupted is a file/store format error. +type ECorrupted struct { + Name string + Ofs int64 +} + +func (e *ECorrupted) Error() string { + return fmt.Sprintf("%s: corrupted data @%#x", e.Name, e.Ofs) +} + +// ECreate is a file/store create error. +type ECreate struct { + Name string + Err error +} + +func (e *ECreate) Error() string { + return fmt.Sprintf("%s: %s", e.Name, e.Err) +} + +// EFreeList is a file/store format error. +type EFreeList struct { + Name string + Size int64 + Block int64 +} + +func (e *EFreeList) Error() string { + return fmt.Sprintf("%s: invalid free list item, size %#x, block %#x", e.Name, e.Size, e.Block) +} + +// EHandle is an error type reported for invalid Handles. +type EHandle struct { + Name string + Handle Handle +} + +func (e EHandle) Error() string { + return fmt.Sprintf("%s: invalid handle %#x", e.Name, e.Handle) +} + +// EHeader is a file/store format error. +type EHeader struct { + Name string + Header []byte + Expected []byte +} + +func (e *EHeader) Error() string { + return fmt.Sprintf("%s: invalid header, got [% x], expected [% x]", e.Name, e.Header, e.Expected) +} + +// ENullHandle is a file/store access error via a null handle. +type ENullHandle string + +func (e ENullHandle) Error() string { + return fmt.Sprintf("%s: access via null handle", e) +} + +// EOpen is a file/store open error. +type EOpen struct { + Name string + Err error +} + +func (e *EOpen) Error() string { + return fmt.Sprintf("%s: %s", e.Name, e.Err) +} + +// ERead is a file/store read error. +type ERead struct { + Name string + Ofs int64 + Err error +} + +func (e *ERead) Error() string { + return fmt.Sprintf("%s, %#x: %s", e.Name, e.Ofs, e.Err) +} + +// ESize is a file/store size error. +type ESize struct { + Name string + Size int64 +} + +func (e *ESize) Error() string { + return fmt.Sprintf("%s: invalid size %#x(%d), size %%16 != 0", e.Name, e.Size, e.Size) +} + +// EWrite is a file/store write error. +type EWrite struct { + Name string + Ofs int64 + Err error +} + +func (e *EWrite) Error() string { + return fmt.Sprintf("%s, %#x: %s", e.Name, e.Ofs, e.Err) +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/falloc.go b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/falloc.go new file mode 100644 index 00000000000..066a7047f3d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/falloc.go @@ -0,0 +1,676 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +/* + +This is an mostly (WIP) conforming implementation of the "specs" in docs.go. + +The main incompletness is support for only one kind of FTL, though this table kind is still per "specs". + +*/ + +package falloc + +import ( + "bytes" + "github.com/cznic/fileutil/storage" + "sync" +) + +// Handle is a reference to a block in a file/store. +// Handle is an uint56 wrapped in an in64, i.e. the most significant byte must be always zero. +type Handle int64 + +// Put puts the 7 least significant bytes of h into b. The MSB of h should be zero. +func (h Handle) Put(b []byte) { + for ofs := 6; ofs >= 0; ofs-- { + b[ofs] = byte(h) + h >>= 8 + } +} + +// Get gets the 7 least significant bytes of h from b. The MSB of h is zeroed. +func (h *Handle) Get(b []byte) { + var x Handle + for ofs := 0; ofs <= 6; ofs++ { + x = x<<8 | Handle(b[ofs]) + } + *h = x +} + +// File is a file/store with space allocation/deallocation support. +type File struct { + f storage.Accessor + atoms int64 // current file size in atom units + canfree int64 // only blocks >= canfree can be subject to Free() + freetab [3857]int64 // freetab[0] is unused, freetab[1] is size 1 ptr, freetab[2] is size 2 ptr, ... + rwm sync.RWMutex +} + +func (f *File) read(b []byte, off int64) { + if n, err := f.f.ReadAt(b, off); n != len(b) { + panic(&ERead{f.f.Name(), off, err}) + } +} + +func (f *File) write(b []byte, off int64) { + if n, err := f.f.WriteAt(b, off); n != len(b) { + panic(&EWrite{f.f.Name(), off, err}) + } +} + +var ( // R/O + hdr = []byte{0x0f, 0xf1, 0xc1, 0xa1, 0xfe, 0xa5, 0x1b, 0x1e, 0, 0, 0, 0, 0, 0, 2, 0} // free lists table @2 + empty = make([]byte, 16) + zero = []byte{0} + zero7 = make([]byte, 7) +) + +// New returns a new File backed by store or an error if any. +// Any existing data in store are discarded. +func New(store storage.Accessor) (f *File, err error) { + f = &File{f: store} + return f, storage.Mutate(store, func() (err error) { + if err = f.f.Truncate(0); err != nil { + return &ECreate{f.f.Name(), err} + } + + if _, err = f.Alloc(hdr[1:]); err != nil { //TODO internal panicking versions of the exported fns. + return + } + + if _, err = f.Alloc(nil); err != nil { // (empty) root @1 + return + } + + b := make([]byte, 3856*14) + for i := 1; i <= 3856; i++ { + Handle(i).Put(b[(i-1)*14:]) + } + if _, err = f.Alloc(b); err != nil { + return + } + + f.canfree = f.atoms + return + }) +} + +// Open returns a new File backed by store or an error if any. +// Store already has to be in a valid format. +func Open(store storage.Accessor) (f *File, err error) { + defer func() { + if e := recover(); e != nil { + f = nil + err = e.(error) + } + }() + + fi, err := store.Stat() + if err != nil { + panic(&EOpen{store.Name(), err}) + } + + fs := fi.Size() + if fs&0xf != 0 { + panic(&ESize{store.Name(), fi.Size()}) + } + + f = &File{f: store, atoms: fs >> 4} + b := make([]byte, len(hdr)) + f.read(b, 0) + if !bytes.Equal(b, hdr) { + panic(&EHeader{store.Name(), b, append([]byte{}, hdr...)}) + } + + var atoms int64 + b, atoms = f.readUsed(2) + f.canfree = atoms + 2 + ofs := 0 + var size, p Handle + for ofs < len(b) { + size.Get(b[ofs:]) + ofs += 7 + p.Get(b[ofs:]) + ofs += 7 + if sz, pp := int64(size), int64(p); size == 0 || size > 3856 || (pp != 0 && pp < f.canfree) || pp<<4 > fs-16 { + panic(&EFreeList{store.Name(), sz, pp}) + } + + f.freetab[size] = int64(p) + } + return +} + +// Accessor returns the File's underlying Accessor. +func (f *File) Accessor() storage.Accessor { + return f.f +} + +// Close closes f and returns an error if any. +func (f *File) Close() (err error) { + return storage.Mutate(f.Accessor(), func() (err error) { + if err = f.f.Close(); err != nil { + err = &EClose{f.f.Name(), err} + } + return + }) +} + +// Root returns the handle of the DB root (top level directory, ...). +func (f *File) Root() Handle { + return 1 +} + +func (f *File) readUsed(atom int64) (content []byte, atoms int64) { + b, redirected := make([]byte, 7), false +redir: + ofs := atom << 4 + f.read(b[:1], ofs) + switch pre := b[0]; { + default: + panic(&ECorrupted{f.f.Name(), ofs}) + case pre == 0x00: // Empty block + case pre >= 1 && pre <= 237: // Short + content = make([]byte, pre) + f.read(content, ofs+1) + case pre >= 0xee && pre <= 0xfb: // Short esc + content = make([]byte, 15+16*(pre-0xee)) + f.read(content, ofs+1) + content[len(content)-1] += 0xfe + case pre == 0xfc: // Long + f.read(b[:2], ofs+1) + n := int(b[0])<<8 + int(b[1]) + switch { + default: + panic(&ECorrupted{f.f.Name(), ofs + 1}) + case n >= 238 && n <= 61680: // Long non esc + content = make([]byte, n) + f.read(content, ofs+3) + case n >= 61681: // Long esc + content = make([]byte, 13+16*(n-0xf0f1)) + f.read(content, ofs+3) + content[len(content)-1] += 0xfe + } + case pre == 0xfd: // redir + if redirected { + panic(&ECorrupted{f.f.Name(), ofs}) + } + + f.read(b[:7], ofs+1) + (*Handle)(&atom).Get(b) + redirected = true + goto redir + } + return content, rq2Atoms(len(content)) +} + +func (f *File) writeUsed(b []byte, atom int64) { + n := len(b) + switch ofs, atoms, endmark := atom<<4, rq2Atoms(n), true; { + default: + panic("internal error") + case n == 0: + f.write(empty, ofs) + case n <= 237: + if (n+1)&0xf == 0 { // content end == atom end + if v := b[n-1]; v >= 0xfe { // escape + pre := []byte{byte((16*0xee + n - 15) >> 4)} + f.write(pre, ofs) + f.write(b[:n-1], ofs+1) + f.write([]byte{v - 0xfe}, ofs+atoms<<4-1) + return + } + endmark = false + } + // non esacpe + pre := []byte{byte(n)} + f.write(pre, ofs) + f.write(b, ofs+1) + if endmark { + f.write(zero, ofs+atoms<<4-1) // last block byte <- used block + } + case n > 237 && n <= 61680: + if (n+3)&0xf == 0 { // content end == atom end + if v := b[n-1]; v >= 0xfe { // escape + x := (16*0xf0f1 + n - 13) >> 4 + pre := []byte{0xFC, byte(x >> 8), byte(x)} + f.write(pre, ofs) + f.write(b[:n-1], ofs+3) + f.write([]byte{v - 0xfe}, ofs+atoms<<4-1) + return + } + endmark = false + } + // non esacpe + pre := []byte{0xfc, byte(n >> 8), byte(n)} + f.write(pre, ofs) + f.write(b, ofs+3) + if endmark { + f.write(zero, ofs+atoms<<4-1) // last block byte <- used block + } + } +} + +func rq2Atoms(rqbytes int) (rqatoms int64) { + if rqbytes > 237 { + rqbytes += 2 + } + return int64(rqbytes>>4 + 1) +} + +func (f *File) extend(b []byte) (handle int64) { + handle = f.atoms + f.writeUsed(b, handle) + f.atoms += rq2Atoms(len(b)) + return +} + +// Alloc stores b in a newly allocated space and returns its handle and an error if any. +func (f *File) Alloc(b []byte) (handle Handle, err error) { + err = storage.Mutate(f.Accessor(), func() (err error) { + rqAtoms := rq2Atoms(len(b)) + if rqAtoms > 3856 { + return &EBadRequest{f.f.Name(), len(b)} + } + + for foundsize, foundp := range f.freetab[rqAtoms:] { + if foundp != 0 { + // this works only for the current unique sizes list (except the last item!) + size := int64(foundsize) + rqAtoms + handle = Handle(foundp) + if size == 3856 { + buf := make([]byte, 7) + f.read(buf, int64(handle)<<4+15) + (*Handle)(&size).Get(buf) + } + f.delFree(int64(handle), size) + if rqAtoms < size { + f.addFree(int64(handle)+rqAtoms, size-rqAtoms) + } + f.writeUsed(b, int64(handle)) + return + } + } + + handle = Handle(f.extend(b)) + return + }) + return +} + +// checkLeft returns the atom size of a free bleck left adjacent to block @atom. +// If that block is not free the returned size is 0. +func (f *File) checkLeft(atom int64) (size int64) { + if atom <= f.canfree { + return + } + + b := make([]byte, 7) + fp := atom << 4 + f.read(b[:1], fp-1) + switch last := b[0]; { + case last <= 0xfd: + // used block + case last == 0xfe: + f.read(b, fp-8) + (*Handle)(&size).Get(b) + case last == 0xff: + size = 1 + } + return +} + +// getInfo returns the block @atom type and size. +func (f *File) getInfo(atom int64) (pref byte, size int64) { + b := make([]byte, 7) + fp := atom << 4 + f.read(b[:1], fp) + switch pref = b[0]; { + case pref == 0: // Empty used + size = 1 + case pref >= 1 && pref <= 237: // Short + size = rq2Atoms(int(pref)) + case pref >= 0xee && pref <= 0xfb: // Short esc + size = rq2Atoms(15 + 16*int(pref-0xee)) + case pref == 0xfc: // Long + f.read(b[:2], fp+1) + n := int(b[0])<<8 + int(b[1]) + switch { + default: + panic(&ECorrupted{f.f.Name(), fp + 1}) + case n >= 238 && n <= 61680: // Long non esc + size = rq2Atoms(n) + case n >= 61681: // Long esc + size = rq2Atoms(13 + 16*(n-0xf0f1)) + } + case pref == 0xfd: // reloc + size = 1 + case pref == 0xfe: + f.read(b, fp+15) + (*Handle)(&size).Get(b) + case pref == 0xff: + size = 1 + } + return +} + +// getSize returns the atom size of the block @atom and wheter it is free. +func (f *File) getSize(atom int64) (size int64, isFree bool) { + var typ byte + typ, size = f.getInfo(atom) + isFree = typ >= 0xfe + return +} + +// checkRight returns the atom size of a free bleck right adjacent to block @atom,atoms. +// If that block is not free the returned size is 0. +func (f *File) checkRight(atom, atoms int64) (size int64) { + if atom+atoms >= f.atoms { + return + } + + if sz, free := f.getSize(atom + atoms); free { + size = sz + } + return +} + +// delFree removes the atoms@atom free block from the free block list +func (f *File) delFree(atom, atoms int64) { + b := make([]byte, 15) + size := int(atoms) + if n := len(f.freetab); atoms >= int64(n) { + size = n - 1 + } + fp := atom << 4 + f.read(b[1:], fp+1) + var prev, next Handle + prev.Get(b[1:]) + next.Get(b[8:]) + + switch { + case prev == 0 && next != 0: + next.Put(b) + f.write(b[:7], int64(32+3+7+(size-1)*14)) + f.write(zero7, int64(next)<<4+1) + f.freetab[size] = int64(next) + case prev != 0 && next == 0: + f.write(zero7, int64(prev)<<4+8) + case prev != 0 && next != 0: + prev.Put(b) + f.write(b[:7], int64(next)<<4+1) + next.Put(b) + f.write(b[:7], int64(prev)<<4+8) + default: // prev == 0 && next == 0: + f.write(zero7, int64(32+3+7+(size-1)*14)) + f.freetab[size] = 0 + } +} + +// addFree adds atoms@atom to the free block lists and marks it free. +func (f *File) addFree(atom, atoms int64) { + b := make([]byte, 7) + size := int(atoms) + if n := len(f.freetab); atoms >= int64(n) { + size = n - 1 + } + head := f.freetab[size] + if head == 0 { // empty list + f.makeFree(0, atom, atoms, 0) + Handle(atom).Put(b) + f.write(b, int64(32+3+7+(size-1)*14)) + f.freetab[size] = atom + return + } + + Handle(atom).Put(b) + f.write(b, head<<4+1) // head.prev = atom + f.makeFree(0, atom, atoms, head) // atom.next = head + f.write(b, int64(32+3+7+(size-1)*14)) + f.freetab[size] = atom +} + +// makeFree sets up the content of a free block atoms@atom, fills the prev and next links. +func (f *File) makeFree(prev, atom, atoms, next int64) { + b := make([]byte, 23) + fp := atom << 4 + if atoms == 1 { + b[0] = 0xff + Handle(prev).Put(b[1:]) + Handle(next).Put(b[8:]) + b[15] = 0xff + f.write(b[:16], fp) + return + } + + b[0] = 0xfe + Handle(prev).Put(b[1:]) + Handle(next).Put(b[8:]) + Handle(atoms).Put(b[15:]) + f.write(b[:22], fp) + b[22] = 0xfe + f.write(b[15:], fp+atoms<<4-8) +} + +// Read reads and return the data associated with handle and an error if any. +// Passing an invalid handle to Read may return invalid data without error. +// It's like getting garbage via passing an invalid pointer to C.memcopy(). +func (f *File) Read(handle Handle) (b []byte, err error) { + defer func() { + if e := recover(); e != nil { + b = nil + err = e.(error) + } + }() + + switch handle { + case 0: + panic(ENullHandle(f.f.Name())) + case 2: + panic(&EHandle{f.f.Name(), handle}) + default: + b, _ = f.readUsed(int64(handle)) + } + return +} + +// Free frees space associated with handle and returns an error if any. Passing an invalid +// handle to Free or reusing handle afterwards will probably corrupt the database or provide +// invalid data on Read. It's like corrupting memory via passing an invalid pointer to C.free() +// or reusing that pointer. +func (f *File) Free(handle Handle) (err error) { + return storage.Mutate(f.Accessor(), func() (err error) { + atom := int64(handle) + atoms, isFree := f.getSize(atom) + if isFree || atom < f.canfree { + return &EHandle{f.f.Name(), handle} + } + + leftFree, rightFree := f.checkLeft(atom), f.checkRight(atom, atoms) + switch { + case leftFree != 0 && rightFree != 0: + f.delFree(atom-leftFree, leftFree) + f.delFree(atom+atoms, rightFree) + f.addFree(atom-leftFree, leftFree+atoms+rightFree) + case leftFree != 0 && rightFree == 0: + f.delFree(atom-leftFree, leftFree) + if atom+atoms == f.atoms { // the left free neighbour and this block together are an empy tail + f.atoms = atom - leftFree + f.f.Truncate(f.atoms << 4) + return + } + + f.addFree(atom-leftFree, leftFree+atoms) + case leftFree == 0 && rightFree != 0: + f.delFree(atom+atoms, rightFree) + f.addFree(atom, atoms+rightFree) + default: // leftFree == 0 && rightFree == 0 + if atom+atoms < f.atoms { // isolated inner block + f.addFree(atom, atoms) + return + } + + f.f.Truncate(atom << 4) // isolated tail block, shrink file + f.atoms = atom + } + return + }) +} + +// Realloc reallocates space associted with handle to acomodate b, returns the newhandle +// newly associated with b and an error if any. If keepHandle == true then Realloc guarantees +// newhandle == handle even if the new data are larger then the previous content associated +// with handle. If !keepHandle && newhandle != handle then reusing handle will probably corrupt +// the database. +// The above effects are like corrupting memory/data via passing an invalid pointer to C.realloc(). +func (f *File) Realloc(handle Handle, b []byte, keepHandle bool) (newhandle Handle, err error) { + err = storage.Mutate(f.Accessor(), func() (err error) { + switch handle { + case 0, 2: + return &EHandle{f.f.Name(), handle} + case 1: + keepHandle = true + } + newhandle = handle + atom, newatoms := int64(handle), rq2Atoms(len(b)) + if newatoms > 3856 { + return &EBadRequest{f.f.Name(), len(b)} + } + + typ, oldatoms := f.getInfo(atom) + switch { + default: + return &ECorrupted{f.f.Name(), atom << 4} + case typ <= 0xfc: // non relocated used block + switch { + case newatoms == oldatoms: // in place replace + f.writeUsed(b, atom) + case newatoms < oldatoms: // in place shrink + rightFree := f.checkRight(atom, oldatoms) + if rightFree > 0 { // right join + f.delFree(atom+oldatoms, rightFree) + } + f.addFree(atom+newatoms, oldatoms+rightFree-newatoms) + f.writeUsed(b, atom) + case newatoms > oldatoms: + if rightFree := f.checkRight(atom, oldatoms); rightFree > 0 && newatoms <= oldatoms+rightFree { + f.delFree(atom+oldatoms, rightFree) + if newatoms < oldatoms+rightFree { + f.addFree(atom+newatoms, oldatoms+rightFree-newatoms) + } + f.writeUsed(b, atom) + return + } + + if !keepHandle { + f.Free(Handle(atom)) + newhandle, err = f.Alloc(b) + return + } + + // reloc + newatom, e := f.Alloc(b) + if e != nil { + return e + } + + buf := make([]byte, 16) + buf[0] = 0xfd + Handle(newatom).Put(buf[1:]) + f.Realloc(Handle(atom), buf[1:], true) + f.write(buf[:1], atom<<4) + } + case typ == 0xfd: // reloc + var target Handle + buf := make([]byte, 7) + f.read(buf, atom<<4+1) + target.Get(buf) + switch { + case newatoms == 1: + f.writeUsed(b, atom) + f.Free(target) + default: + if rightFree := f.checkRight(atom, 1); rightFree > 0 && newatoms <= 1+rightFree { + f.delFree(atom+1, rightFree) + if newatoms < 1+rightFree { + f.addFree(atom+newatoms, 1+rightFree-newatoms) + } + f.writeUsed(b, atom) + f.Free(target) + return + } + + newtarget, e := f.Realloc(Handle(target), b, false) + if e != nil { + return e + } + + if newtarget != target { + Handle(newtarget).Put(buf) + f.write(buf, atom<<4+1) + } + } + } + return + }) + return +} + +// Lock locks f for writing. If the lock is already locked for reading or writing, +// Lock blocks until the lock is available. To ensure that the lock eventually becomes available, +// a blocked Lock call excludes new readers from acquiring the lock. +func (f *File) Lock() { + f.rwm.Lock() +} + +// RLock locks f for reading. If the lock is already locked for writing or there is a writer +// already waiting to release the lock, RLock blocks until the writer has released the lock. +func (f *File) RLock() { + f.rwm.RLock() +} + +// Unlock unlocks f for writing. It is a run-time error if f is not locked for writing on entry to Unlock. +// +// As with Mutexes, a locked RWMutex is not associated with a particular goroutine. +// One goroutine may RLock (Lock) f and then arrange for another goroutine to RUnlock (Unlock) it. +func (f *File) Unlock() { + f.rwm.Unlock() +} + +// RUnlock undoes a single RLock call; it does not affect other simultaneous readers. +// It is a run-time error if f is not locked for reading on entry to RUnlock. +func (f *File) RUnlock() { + f.rwm.RUnlock() +} + +// LockedAlloc wraps Alloc in a Lock/Unlock pair. +func (f *File) LockedAlloc(b []byte) (handle Handle, err error) { + f.Lock() + defer f.Unlock() + return f.Alloc(b) +} + +// LockedFree wraps Free in a Lock/Unlock pair. +func (f *File) LockedFree(handle Handle) (err error) { + f.Lock() + defer f.Unlock() + return f.Free(handle) +} + +// LockedRead wraps Read in a RLock/RUnlock pair. +func (f *File) LockedRead(handle Handle) (b []byte, err error) { + f.RLock() + defer f.RUnlock() + return f.Read(handle) +} + +// LockedRealloc wraps Realloc in a Lock/Unlock pair. +func (f *File) LockedRealloc(handle Handle, b []byte, keepHandle bool) (newhandle Handle, err error) { + f.Lock() + defer f.Unlock() + return f.Realloc(handle, b, keepHandle) +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/test_deps.go b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/test_deps.go new file mode 100644 index 00000000000..3bdd39f2b56 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/falloc/test_deps.go @@ -0,0 +1,15 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package falloc + +// Pull test dependencies too. +// Enables easy 'go test X' after 'go get X' +import ( + _ "github.com/cznic/fileutil" + _ "github.com/cznic/fileutil/storage" + _ "github.com/cznic/mathutil" +) diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil.go new file mode 100644 index 00000000000..2f0f7ab155d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil.go @@ -0,0 +1,223 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fileutil collects some file utility functions. +package fileutil + +import ( + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strconv" + "sync" + "time" +) + +// GoMFile is a concurrent access safe version of MFile. +type GoMFile struct { + mfile *MFile + mutex sync.Mutex +} + +// NewGoMFile return a newly created GoMFile. +func NewGoMFile(fname string, flag int, perm os.FileMode, delta_ns int64) (m *GoMFile, err error) { + m = &GoMFile{} + if m.mfile, err = NewMFile(fname, flag, perm, delta_ns); err != nil { + m = nil + } + return +} + +func (m *GoMFile) File() (file *os.File, err error) { + m.mutex.Lock() + defer m.mutex.Unlock() + return m.mfile.File() +} + +func (m *GoMFile) SetChanged() { + m.mutex.Lock() + defer m.mutex.Unlock() + m.mfile.SetChanged() +} + +func (m *GoMFile) SetHandler(h MFileHandler) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.mfile.SetHandler(h) +} + +// MFileHandler resolves modifications of File. +// Possible File context is expected to be a part of the handler's closure. +type MFileHandler func(*os.File) error + +// MFile represents an os.File with a guard/handler on change/modification. +// Example use case is an app with a configuration file which can be modified at any time +// and have to be reloaded in such event prior to performing something configurable by that +// file. The checks are made only on access to the MFile file by +// File() and a time threshold/hysteresis value can be chosen on creating a new MFile. +type MFile struct { + file *os.File + handler MFileHandler + t0 int64 + delta int64 + ctime int64 +} + +// NewMFile returns a newly created MFile or Error if any. +// The fname, flag and perm parameters have the same meaning as in os.Open. +// For meaning of the delta_ns parameter please see the (m *MFile) File() docs. +func NewMFile(fname string, flag int, perm os.FileMode, delta_ns int64) (m *MFile, err error) { + m = &MFile{} + m.t0 = time.Now().UnixNano() + if m.file, err = os.OpenFile(fname, flag, perm); err != nil { + return + } + + var fi os.FileInfo + if fi, err = m.file.Stat(); err != nil { + return + } + + m.ctime = fi.ModTime().UnixNano() + m.delta = delta_ns + runtime.SetFinalizer(m, func(m *MFile) { + m.file.Close() + }) + return +} + +// SetChanged forces next File() to unconditionally handle modification of the wrapped os.File. +func (m *MFile) SetChanged() { + m.ctime = -1 +} + +// SetHandler sets a function to be invoked when modification of MFile is to be processed. +func (m *MFile) SetHandler(h MFileHandler) { + m.handler = h +} + +// File returns an os.File from MFile. If time elapsed between the last invocation of this function +// and now is at least delta_ns ns (a parameter of NewMFile) then the file is checked for +// change/modification. For delta_ns == 0 the modification is checked w/o getting os.Time(). +// If a change is detected a handler is invoked on the MFile file. +// Any of these steps can produce an Error. If that happens the function returns nil, Error. +func (m *MFile) File() (file *os.File, err error) { + var now int64 + + mustCheck := m.delta == 0 + if !mustCheck { + now = time.Now().UnixNano() + mustCheck = now-m.t0 > m.delta + } + + if mustCheck { // check interval reached + var fi os.FileInfo + if fi, err = m.file.Stat(); err != nil { + return + } + + if fi.ModTime().UnixNano() != m.ctime { // modification detected + if m.handler == nil { + return nil, fmt.Errorf("no handler set for modified file %q", m.file.Name()) + } + if err = m.handler(m.file); err != nil { + return + } + + m.ctime = fi.ModTime().UnixNano() + } + m.t0 = now + } + + return m.file, nil +} + +// Read reads buf from r. It will either fill the full buf or fail. +// It wraps the functionality of an io.Reader which may return less bytes than requested, +// but may block if not all data are ready for the io.Reader. +func Read(r io.Reader, buf []byte) (err error) { + have := 0 + remain := len(buf) + got := 0 + for remain > 0 { + if got, err = r.Read(buf[have:]); err != nil { + return + } + + remain -= got + have += got + } + return +} + +// "os" and/or "syscall" extensions + +// FadviseAdvice is used by Fadvise. +type FadviseAdvice int + +// FAdviseAdvice values. +const ( + // $ grep FADV /usr/include/bits/fcntl.h + POSIX_FADV_NORMAL FadviseAdvice = iota // No further special treatment. + POSIX_FADV_RANDOM // Expect random page references. + POSIX_FADV_SEQUENTIAL // Expect sequential page references. + POSIX_FADV_WILLNEED // Will need these pages. + POSIX_FADV_DONTNEED // Don't need these pages. + POSIX_FADV_NOREUSE // Data will be accessed once. +) + +// TempFile creates a new temporary file in the directory dir with a name +// ending with suffix, basename starting with prefix, opens the file for +// reading and writing, and returns the resulting *os.File. If dir is the +// empty string, TempFile uses the default directory for temporary files (see +// os.TempDir). Multiple programs calling TempFile simultaneously will not +// choose the same file. The caller can use f.Name() to find the pathname of +// the file. It is the caller's responsibility to remove the file when no +// longer needed. +// +// NOTE: This function differs from ioutil.TempFile. +func TempFile(dir, prefix, suffix string) (f *os.File, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextInfix()+suffix) + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + rand = reseed() + } + continue + } + break + } + return +} + +// Random number state. +// We generate random temporary file names so that there's a good +// chance the file doesn't exist yet - keeps the number of tries in +// TempFile to a minimum. +var rand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextInfix() string { + randmu.Lock() + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_arm.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_arm.go new file mode 100644 index 00000000000..9410d1bb26d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_arm.go @@ -0,0 +1,25 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "io" + "os" +) + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. Not supported on ARM. +func PunchHole(f *os.File, off, len int64) error { + return nil +} + +// Fadvise predeclares an access pattern for file data. See also 'man 2 +// posix_fadvise'. Not supported on ARM. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + return nil +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { return err == io.EOF } diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_darwin.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_darwin.go new file mode 100644 index 00000000000..a19723fc78f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_darwin.go @@ -0,0 +1,25 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "io" + "os" +) + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. Not supported on OSX. +func PunchHole(f *os.File, off, len int64) error { + return nil +} + +// Fadvise predeclares an access pattern for file data. See also 'man 2 +// posix_fadvise'. Not supported on OSX. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + return nil +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { return err == io.EOF } diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_freebsd.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_freebsd.go new file mode 100644 index 00000000000..0865093e2bd --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_freebsd.go @@ -0,0 +1,25 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "io" + "os" +) + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. Unimplemented on FreeBSD. +func PunchHole(f *os.File, off, len int64) error { + return nil +} + +// Fadvise predeclares an access pattern for file data. See also 'man 2 +// posix_fadvise'. Unimplemented on FreeBSD. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + return nil +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { return err == io.EOF } diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_linux.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_linux.go new file mode 100644 index 00000000000..8babfc55335 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_linux.go @@ -0,0 +1,96 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !arm + +package fileutil + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "strconv" + "syscall" +) + +func n(s []byte) byte { + for i, c := range s { + if c < '0' || c > '9' { + s = s[:i] + break + } + } + v, _ := strconv.Atoi(string(s)) + return byte(v) +} + +func init() { + b, err := ioutil.ReadFile("/proc/sys/kernel/osrelease") + if err != nil { + panic(err) + } + + tokens := bytes.Split(b, []byte(".")) + if len(tokens) > 3 { + tokens = tokens[:3] + } + switch len(tokens) { + case 3: + // Supported since kernel 2.6.38 + if bytes.Compare([]byte{n(tokens[0]), n(tokens[1]), n(tokens[2])}, []byte{2, 6, 38}) < 0 { + puncher = func(*os.File, int64, int64) error { return nil } + } + case 2: + if bytes.Compare([]byte{n(tokens[0]), n(tokens[1])}, []byte{2, 7}) < 0 { + puncher = func(*os.File, int64, int64) error { return nil } + } + default: + puncher = func(*os.File, int64, int64) error { return nil } + } +} + +var puncher = func(f *os.File, off, len int64) error { + const ( + /* + /usr/include/linux$ grep FL_ falloc.h + */ + _FALLOC_FL_KEEP_SIZE = 0x01 // default is extend size + _FALLOC_FL_PUNCH_HOLE = 0x02 // de-allocates range + ) + + _, _, errno := syscall.Syscall6( + syscall.SYS_FALLOCATE, + uintptr(f.Fd()), + uintptr(_FALLOC_FL_KEEP_SIZE|_FALLOC_FL_PUNCH_HOLE), + uintptr(off), + uintptr(len), + 0, 0) + if errno != 0 { + return os.NewSyscallError("SYS_FALLOCATE", errno) + } + return nil +} + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. No-op for kernels < 2.6.38 (or < 2.7). +func PunchHole(f *os.File, off, len int64) error { + return puncher(f, off, len) +} + +// Fadvise predeclares an access pattern for file data. See also 'man 2 +// posix_fadvise'. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + _, _, errno := syscall.Syscall6( + syscall.SYS_FADVISE64, + uintptr(f.Fd()), + uintptr(off), + uintptr(len), + uintptr(advice), + 0, 0) + return os.NewSyscallError("SYS_FADVISE64", errno) +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { return err == io.EOF } diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_openbsd.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_openbsd.go new file mode 100644 index 00000000000..428171bdfe7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_openbsd.go @@ -0,0 +1,25 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "io" + "os" +) + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. Similar to FreeBSD, this is +// unimplemented. +func PunchHole(f *os.File, off, len int64) error { + return nil +} + +// Unimplemented on OpenBSD. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + return nil +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { return err == io.EOF } diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_plan9.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_plan9.go new file mode 100644 index 00000000000..a2db64e2d3b --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_plan9.go @@ -0,0 +1,25 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "io" + "os" +) + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. Unimplemented on Plan 9. +func PunchHole(f *os.File, off, len int64) error { + return nil +} + +// Fadvise predeclares an access pattern for file data. See also 'man 2 +// posix_fadvise'. Unimplemented on Plan 9. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + return nil +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { return err == io.EOF } diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_solaris.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_solaris.go new file mode 100644 index 00000000000..61dfcde38f8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_solaris.go @@ -0,0 +1,27 @@ +// Copyright (c) 2013 jnml. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.3 + +package fileutil + +import ( + "io" + "os" +) + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. Not supported on Solaris. +func PunchHole(f *os.File, off, len int64) error { + return nil +} + +// Fadvise predeclares an access pattern for file data. See also 'man 2 +// posix_fadvise'. Not supported on Solaris. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + return nil +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { return err == io.EOF } diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_windows.go b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_windows.go new file mode 100644 index 00000000000..3a81f2fc873 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_windows.go @@ -0,0 +1,183 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "io" + "os" + "sync" + "syscall" + "unsafe" +) + +// PunchHole deallocates space inside a file in the byte range starting at +// offset and continuing for len bytes. Not supported on Windows. +func PunchHole(f *os.File, off, len int64) error { + return puncher(f, off, len) +} + +// Fadvise predeclares an access pattern for file data. See also 'man 2 +// posix_fadvise'. Not supported on Windows. +func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error { + return nil +} + +// IsEOF reports whether err is an EOF condition. +func IsEOF(err error) bool { + if err == io.EOF { + return true + } + + // http://social.technet.microsoft.com/Forums/windowsserver/en-US/1a16311b-c625-46cf-830b-6a26af488435/how-to-solve-error-38-0x26-errorhandleeof-using-fsctlgetretrievalpointers + x, ok := err.(*os.PathError) + return ok && x.Op == "read" && x.Err.(syscall.Errno) == 0x26 +} + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + + procDeviceIOControl = modkernel32.NewProc("DeviceIoControl") + + sparseFilesMu sync.Mutex + sparseFiles map[uintptr]struct{} +) + +func init() { + // sparseFiles is an fd set for already "sparsed" files - according to + // msdn.microsoft.com/en-us/library/windows/desktop/aa364225(v=vs.85).aspx + // the file handles are unique per process. + sparseFiles = make(map[uintptr]struct{}) +} + +// puncHoleWindows punches a hole into the given file starting at offset, +// measuring "size" bytes +// (http://msdn.microsoft.com/en-us/library/windows/desktop/aa364597%28v=vs.85%29.aspx) +func puncher(file *os.File, offset, size int64) error { + if err := ensureFileSparse(file); err != nil { + return err + } + + // http://msdn.microsoft.com/en-us/library/windows/desktop/aa364411%28v=vs.85%29.aspx + // typedef struct _FILE_ZERO_DATA_INFORMATION { + // LARGE_INTEGER FileOffset; + // LARGE_INTEGER BeyondFinalZero; + //} FILE_ZERO_DATA_INFORMATION, *PFILE_ZERO_DATA_INFORMATION; + type fileZeroDataInformation struct { + FileOffset, BeyondFinalZero int64 + } + + lpInBuffer := fileZeroDataInformation{ + FileOffset: offset, + BeyondFinalZero: offset + size} + return deviceIOControl(false, file.Fd(), uintptr(unsafe.Pointer(&lpInBuffer)), 16) +} + +// // http://msdn.microsoft.com/en-us/library/windows/desktop/cc948908%28v=vs.85%29.aspx +// type fileSetSparseBuffer struct { +// SetSparse bool +// } + +func ensureFileSparse(file *os.File) (err error) { + fd := file.Fd() + sparseFilesMu.Lock() + if _, ok := sparseFiles[fd]; ok { + sparseFilesMu.Unlock() + return nil + } + + if err = deviceIOControl(true, fd, 0, 0); err == nil { + sparseFiles[fd] = struct{}{} + } + sparseFilesMu.Unlock() + return err +} + +func deviceIOControl(setSparse bool, fd, inBuf, inBufLen uintptr) (err error) { + const ( + //http://source.winehq.org/source/include/winnt.h#L4605 + file_read_data = 1 + file_write_data = 2 + + // METHOD_BUFFERED 0 + method_buffered = 0 + // FILE_ANY_ACCESS 0 + file_any_access = 0 + // FILE_DEVICE_FILE_SYSTEM 0x00000009 + file_device_file_system = 0x00000009 + // FILE_SPECIAL_ACCESS (FILE_ANY_ACCESS) + file_special_access = file_any_access + file_read_access = file_read_data + file_write_access = file_write_data + + // http://source.winehq.org/source/include/winioctl.h + // #define CTL_CODE ( DeviceType, + // Function, + // Method, + // Access ) + // ((DeviceType) << 16) | ((Access) << 14) | ((Function) << 2) | (Method) + + // FSCTL_SET_COMPRESSION CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 16, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA) + fsctl_set_compression = (file_device_file_system << 16) | ((file_read_access | file_write_access) << 14) | (16 << 2) | method_buffered + // FSCTL_SET_SPARSE CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 49, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) + fsctl_set_sparse = (file_device_file_system << 16) | (file_special_access << 14) | (49 << 2) | method_buffered + // FSCTL_SET_ZERO_DATA CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 50, METHOD_BUFFERED, FILE_WRITE_DATA) + fsctl_set_zero_data = (file_device_file_system << 16) | (file_write_data << 14) | (50 << 2) | method_buffered + ) + retPtr := uintptr(unsafe.Pointer(&(make([]byte, 8)[0]))) + var r1 uintptr + var e1 syscall.Errno + if setSparse { + // BOOL + // WINAPI + // DeviceIoControl( (HANDLE) hDevice, // handle to a file + // FSCTL_SET_SPARSE, // dwIoControlCode + // (PFILE_SET_SPARSE_BUFFER) lpInBuffer, // input buffer + // (DWORD) nInBufferSize, // size of input buffer + // NULL, // lpOutBuffer + // 0, // nOutBufferSize + // (LPDWORD) lpBytesReturned, // number of bytes returned + // (LPOVERLAPPED) lpOverlapped ); // OVERLAPPED structure + r1, _, e1 = syscall.Syscall9(procDeviceIOControl.Addr(), 8, + fd, + uintptr(fsctl_set_sparse), + // If the lpInBuffer parameter is NULL, the operation will behave the same as if the SetSparse member of the FILE_SET_SPARSE_BUFFER structure were TRUE. In other words, the operation sets the file to a sparse file. + 0, // uintptr(unsafe.Pointer(&lpInBuffer)), + 0, // 1, + 0, + 0, + retPtr, + 0, + 0) + } else { + // BOOL + // WINAPI + // DeviceIoControl( (HANDLE) hDevice, // handle to a file + // FSCTL_SET_ZERO_DATA, // dwIoControlCode + // (LPVOID) lpInBuffer, // input buffer + // (DWORD) nInBufferSize, // size of input buffer + // NULL, // lpOutBuffer + // 0, // nOutBufferSize + // (LPDWORD) lpBytesReturned, // number of bytes returned + // (LPOVERLAPPED) lpOverlapped ); // OVERLAPPED structure + r1, _, e1 = syscall.Syscall9(procDeviceIOControl.Addr(), 8, + fd, + uintptr(fsctl_set_zero_data), + inBuf, + inBufLen, + 0, + 0, + retPtr, + 0, + 0) + } + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return err +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/LICENSE b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/LICENSE new file mode 100644 index 00000000000..1e92e33dd70 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of CZ.NIC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/README b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/README new file mode 100644 index 00000000000..05ce89f92ef --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/README @@ -0,0 +1,5 @@ +This is a goinstall-able mirror of modified code already published at: +https://git.nic.cz/redmine/projects/gofileutil/repository/show/hdb + +Install: $go get github.com/cznic/fileutil/hdb +Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/hdb diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/all_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/all_test.go new file mode 100644 index 00000000000..a980587f1c6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/all_test.go @@ -0,0 +1,15 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package hdb + +import ( + "testing" +) + +func TestPlaceholder(t *testing.T) { + t.Log("TODO") //TODO +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/hdb.go b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/hdb.go new file mode 100644 index 00000000000..7c7113d9d3d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/hdb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +/* +WIP: Package hdb provides a "handle"/value DB like store, but actually it's +closer to the model of a process's virtual memory and its alloc, free and move +methods. + +The hdb package is a thin layer around falloc.File providing stable-only +handles and the basic synchronizing primitives. The central functionality of +hdb are the New, Set, Get and Delete methods of Store. + +Conceptual analogy: + New alloc(sizeof(content)), return new "memory" pointer (a handle). + + Get memmove() from "memory" "pointed to" by handle to the result content. + Note: Handle "knows" the size of its content. + + Set memmove() from content to "memory" pointed to by handle. + In contrast to real memory, the new content may have different + size than the previously stored one w/o additional handling + and the "pointer" handle remains the same. + + Delete free() the "memory" "pointed to" by handle. +*/ +package hdb + +import ( + "github.com/cznic/fileutil/falloc" + "github.com/cznic/fileutil/storage" +) + +type Store struct { + f *falloc.File +} + +// New returns a newly created Store backed by accessor, discarding its conents if any. +// If successful, methods on the returned Store can be used for I/O. +// It returns the Store and an error, if any. +func New(accessor storage.Accessor) (store *Store, err error) { + s := &Store{} + if s.f, err = falloc.New(accessor); err == nil { + store = s + } + return +} + +// Open opens the Store from accessor. +// If successful, methods on the returned Store can be used for data exchange. +// It returns the Store and an error, if any. +func Open(accessor storage.Accessor) (store *Store, err error) { + s := &Store{} + if s.f, err = falloc.Open(accessor); err == nil { + store = s + } + return +} + +// Close closes the store. Further access to the store has undefined behavior and may panic. +// It returns an error, if any. +func (s *Store) Close() (err error) { + defer func() { + s.f = nil + }() + + return s.f.Close() +} + +// Delete deletes the data associated with handle. +// It returns an error if any. +func (s *Store) Delete(handle falloc.Handle) (err error) { + return s.f.Free(handle) +} + +// Get gets the data associated with handle. +// It returns the data and an error, if any. +func (s *Store) Get(handle falloc.Handle) (b []byte, err error) { + return s.f.Read(handle) +} + +// New associates data with a new handle. +// It returns the handle and an error, if any. +func (s *Store) New(b []byte) (handle falloc.Handle, err error) { + return s.f.Alloc(b) +} + +// Set associates data with an existing handle. +// It returns an error, if any. +func (s *Store) Set(handle falloc.Handle, b []byte) (err error) { + _, err = s.f.Realloc(handle, b, true) + return +} + +// Root returns the handle of the DB root (top level directory, ...). +func (s *Store) Root() falloc.Handle { + return s.f.Root() +} + +// File returns the underlying falloc.File of 's'. +func (s *Store) File() *falloc.File { + return s.f +} + +// Lock locks 's' for writing. If the lock is already locked for reading or writing, +// Lock blocks until the lock is available. To ensure that the lock eventually becomes available, +// a blocked Lock call excludes new readers from acquiring the lock. +func (s *Store) Lock() { + s.f.Lock() +} + +// RLock locks 's' for reading. If the lock is already locked for writing or there is a writer +// already waiting to release the lock, RLock blocks until the writer has released the lock. +func (s *Store) RLock() { + s.f.RLock() +} + +// Unlock unlocks 's' for writing. It's a run-time error if 's' is not locked for writing on entry to Unlock. +// +// As with Mutexes, a locked RWMutex is not associated with a particular goroutine. +// One goroutine may RLock (Lock) 's' and then arrange for another goroutine to RUnlock (Unlock) it. +func (s *Store) Unlock() { + s.f.Unlock() +} + +// RUnlock undoes a single RLock call; it does not affect other simultaneous readers. +// It's a run-time error if 's' is not locked for reading on entry to RUnlock. +func (s *Store) RUnlock() { + s.f.RUnlock() +} + +// LockedNew wraps New in a Lock/Unlock pair. +func (s *Store) LockedNew(b []byte) (handle falloc.Handle, err error) { + return s.f.LockedAlloc(b) +} + +// LockedDelete wraps Delete in a Lock/Unlock pair. +func (s *Store) LockedDelete(handle falloc.Handle) (err error) { + return s.f.LockedFree(handle) +} + +// LockedGet wraps Get in a RLock/RUnlock pair. +func (s *Store) LockedGet(handle falloc.Handle) (b []byte, err error) { + return s.f.LockedRead(handle) +} + +// LockedSet wraps Set in a Lock/Unlock pair. +func (s *Store) LockedSet(handle falloc.Handle, b []byte) (err error) { + _, err = s.f.Realloc(handle, b, true) + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/test_deps.go b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/test_deps.go new file mode 100644 index 00000000000..3164f63ae1d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/hdb/test_deps.go @@ -0,0 +1,13 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package hdb + +// Pull test dependencies too. +// Enables easy 'go test X' after 'go get X' +import ( +// nothing yet +) diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/punch_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/punch_test.go new file mode 100644 index 00000000000..766f6f4bae9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/punch_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 The fileutil authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fileutil + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestPunch(t *testing.T) { + file, err := ioutil.TempFile("", "punchhole-") + if err != nil { + t.Error(err) + } + defer os.Remove(file.Name()) + defer file.Close() + buf := make([]byte, 10<<20) + for i := range buf { + buf[i] = byte(1 + (i+1)&0xfe) + } + if _, err = file.Write(buf); err != nil { + t.Errorf("error writing to the temp file: %v", err) + t.FailNow() + } + if err = file.Sync(); err != nil { + t.Logf("error syncing %q: %v", file.Name(), err) + } + for i, j := range []int{1, 31, 1 << 10} { + if err = PunchHole(file, int64(j), int64(j)); err != nil { + t.Errorf("%d. error punching at %d, size %d: %v", i, j, j, err) + continue + } + // read back, with 1-1 bytes overlaid + n, err := file.ReadAt(buf[:j+2], int64(j-1)) + if err != nil { + t.Errorf("%d. error reading file: %v", i, err) + continue + } + buf = buf[:n] + if buf[0] == 0 { + t.Errorf("%d. file at %d has been overwritten with 0!", i, j-1) + } + if buf[n-1] == 0 { + t.Errorf("%d. file at %d has been overwritten with 0!", i, j-1+n) + } + for k, v := range buf[1 : n-1] { + if v != 0 { + t.Errorf("%d. error reading file at %d got %d, want 0.", i, k, v) + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/LICENSE b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/LICENSE new file mode 100644 index 00000000000..1e92e33dd70 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of CZ.NIC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/README b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/README new file mode 100644 index 00000000000..2a400fced65 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/README @@ -0,0 +1,5 @@ +This is a goinstall-able mirror of modified code already published at: +https://git.nic.cz/redmine/projects/gofileutil/repository/show/storage + +Install: $go get github.com/cznic/fileutil/storage +Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/storage diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/all_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/all_test.go new file mode 100644 index 00000000000..8947a0a729c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/all_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "flag" + "runtime" +) + +var ( + devFlag = flag.Bool("dev", false, "enable dev tests") + goFlag = flag.Int("go", 1, "GOMAXPROCS") +) + +func init() { + flag.Parse() + runtime.GOMAXPROCS(*goFlag) +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache.go new file mode 100644 index 00000000000..3a6115a71a3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache.go @@ -0,0 +1,322 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "container/list" + "io" + "math" + "os" + "sync" + "sync/atomic" +) + +type cachepage struct { + b [512]byte + dirty bool + lru *list.Element + pi int64 + valid int // page content is b[:valid] +} + +func (p *cachepage) wr(b []byte, off int) (wasDirty bool) { + copy(p.b[off:], b) + if n := off + len(b); n > p.valid { + p.valid = n + } + wasDirty = p.dirty + p.dirty = true + return +} + +func (c *Cache) rd(off int64, read bool) (p *cachepage, ok bool) { + c.Rq++ + pi := off >> 9 + if p, ok = c.m[pi]; ok { + c.lru.MoveToBack(p.lru) + return + } + + if !read { + return + } + + fp := off &^ 511 + if fp >= c.size { + return + } + + rq := 512 + if fp+512 > c.size { + rq = int(c.size - fp) + } + p = &cachepage{pi: pi, valid: rq} + p.lru = c.lru.PushBack(p) + if n, err := c.f.ReadAt(p.b[:p.valid], fp); n != rq { + panic(err) + } + + c.Load++ + if c.advise != nil { + c.advise(fp, 512, false) + } + c.m[pi], ok = p, true + return +} + +func (c *Cache) wr(off int64) (p *cachepage) { + var ok bool + if p, ok = c.rd(off, false); ok { + return + } + + pi := off >> 9 + p = &cachepage{pi: pi} + p.lru = c.lru.PushBack(p) + c.m[pi] = p + return +} + +// Cache provides caching support for another store Accessor. +type Cache struct { + advise func(int64, int, bool) + clean chan bool + cleaning int32 + close chan bool + f Accessor + fi *FileInfo + lock sync.Mutex + lru *list.List + m map[int64]*cachepage + maxpages int + size int64 + sync chan bool + wlist *list.List + write chan bool + writing int32 + Rq int64 // Pages requested from cache + Load int64 // Pages loaded (cache miss) + Purge int64 // Pages purged + Top int // "High water" pages +} + +// Implementation of Accessor. +func (c *Cache) BeginUpdate() error { return nil } + +// Implementation of Accessor. +func (c *Cache) EndUpdate() error { return nil } + +// NewCache creates a caching Accessor from store with total of maxcache bytes. +// NewCache returns the new Cache, implementing Accessor or an error if any. +// +// The LRU mechanism is used, so the cache tries to keep often accessed pages cached. +// +func NewCache(store Accessor, maxcache int64, advise func(int64, int, bool)) (c *Cache, err error) { + var fi os.FileInfo + if fi, err = store.Stat(); err != nil { + return + } + + x := maxcache >> 9 + if x > math.MaxInt32/2 { + x = math.MaxInt32 / 2 + } + c = &Cache{ + advise: advise, + clean: make(chan bool, 1), + close: make(chan bool), + f: store, + lru: list.New(), // front == oldest used, back == last recently used + m: make(map[int64]*cachepage), + maxpages: int(x), + size: fi.Size(), + sync: make(chan bool), + wlist: list.New(), + write: make(chan bool, 1), + } + c.fi = NewFileInfo(fi, c) + go c.writer() + go c.cleaner(int((int64(c.maxpages) * 95) / 100)) // hysteresis + return +} + +func (c *Cache) Accessor() Accessor { + return c.f +} + +func (c *Cache) Close() (err error) { + close(c.write) + <-c.close + close(c.clean) + <-c.close + return c.f.Close() +} + +func (c *Cache) Name() (s string) { + return c.f.Name() +} + +func (c *Cache) ReadAt(b []byte, off int64) (n int, err error) { + po := int(off) & 0x1ff + bp := 0 + rem := len(b) + m := 0 + for rem != 0 { + c.lock.Lock() // X1+ + p, ok := c.rd(off, true) + if !ok { + c.lock.Unlock() // X1- + return -1, io.EOF + } + + rq := rem + if po+rq > 512 { + rq = 512 - po + } + if n := copy(b[bp:bp+rq], p.b[po:p.valid]); n != rq { + c.lock.Unlock() // X1- + return -1, io.EOF + } + + m = len(c.m) + c.lock.Unlock() // X1- + po = 0 + bp += rq + off += int64(rq) + rem -= rq + n += rq + } + if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) { + if m > c.Top { + c.Top = m + } + c.clean <- true + } + return +} + +func (c *Cache) Stat() (fi os.FileInfo, err error) { + c.lock.Lock() + defer c.lock.Unlock() + return c.fi, nil +} + +func (c *Cache) Sync() (err error) { + c.write <- false + <-c.sync + return +} + +func (c *Cache) Truncate(size int64) (err error) { + c.Sync() //TODO improve (discard pages, the writer goroutine should also be aware, ...) + c.lock.Lock() + defer c.lock.Unlock() + c.size = size + return c.f.Truncate(size) +} + +func (c *Cache) WriteAt(b []byte, off int64) (n int, err error) { + po := int(off) & 0x1ff + bp := 0 + rem := len(b) + m := 0 + for rem != 0 { + c.lock.Lock() // X+ + p := c.wr(off) + rq := rem + if po+rq > 512 { + rq = 512 - po + } + if wasDirty := p.wr(b[bp:bp+rq], po); !wasDirty { + c.wlist.PushBack(p) + } + m = len(c.m) + po = 0 + bp += rq + off += int64(rq) + if off > c.size { + c.size = off + } + c.lock.Unlock() // X- + rem -= rq + n += rq + } + if atomic.CompareAndSwapInt32(&c.writing, 0, 1) { + c.write <- true + } + if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) { + if m > c.Top { + c.Top = m + } + c.clean <- true + } + return +} + +func (c *Cache) writer() { + for ok := true; ok; { + var wr bool + var off int64 + wr, ok = <-c.write + for { + c.lock.Lock() // X1+ + item := c.wlist.Front() + if item == nil { + c.lock.Unlock() // X1- + break + } + + p := item.Value.(*cachepage) + off = p.pi << 9 + if n, err := c.f.WriteAt(p.b[:p.valid], off); n != p.valid { + c.lock.Unlock() // X1- + panic("TODO Cache.writer errchan") //TODO +errchan + panic(err) + } + + p.dirty = false + c.wlist.Remove(item) + if c.advise != nil { + c.advise(off, 512, true) + } + c.lock.Unlock() // X1- + } + switch { + case wr: + atomic.AddInt32(&c.writing, -1) + case ok: + c.sync <- true + } + } + c.close <- true +} + +func (c *Cache) cleaner(limit int) { + for _ = range c.clean { + var item *list.Element + for { + c.lock.Lock() // X1+ + if len(c.m) < limit { + c.lock.Unlock() // X1- + break + } + + if item == nil { + item = c.lru.Front() + } + if p := item.Value.(*cachepage); !p.dirty { + delete(c.m, p.pi) + c.lru.Remove(item) + c.Purge++ + } + item = item.Next() + c.lock.Unlock() // X1- + } + atomic.AddInt32(&c.cleaning, -1) + } + c.close <- true +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache_test.go new file mode 100644 index 00000000000..8dbad08879c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func newfile(t *testing.T) (string, string, Accessor) { + dir, err := ioutil.TempDir("", "test-storage-") + if err != nil { + panic(err) + } + + name := filepath.Join(dir, "test.tmp") + f, err := NewFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666) + if err != nil { + t.Fatal("newfile", err) + } + + return dir, name, f +} + +func readfile(t *testing.T, name string) (b []byte) { + var err error + if b, err = ioutil.ReadFile(name); err != nil { + t.Fatal("readfile") + } + + return +} + +func newcache(t *testing.T) (dir, name string, c *Cache) { + dir, name, f := newfile(t) + var err error + if c, err = NewCache(f, 1<<20, nil); err != nil { + t.Fatal("newCache", err) + } + + return +} + +func TestCache0(t *testing.T) { + dir, name, c := newcache(t) + defer os.RemoveAll(dir) + + if err := c.Close(); err != nil { + t.Fatal(10, err) + } + + if b := readfile(t, name); len(b) != 0 { + t.Fatal(20, len(b), 0) + } +} + +func TestCache1(t *testing.T) { + dir, name, c := newcache(t) + defer os.RemoveAll(dir) + + if n, err := c.WriteAt([]byte{0xa5}, 0); n != 1 { + t.Fatal(20, n, err) + } + + if err := c.Close(); err != nil { + t.Fatal(10, err) + } + + b := readfile(t, name) + if len(b) != 1 { + t.Fatal(30, len(b), 1) + } + + if b[0] != 0xa5 { + t.Fatal(40, b[0], 0xa5) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/dev_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/dev_test.go new file mode 100644 index 00000000000..7287a27ae3c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/dev_test.go @@ -0,0 +1,18 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "testing" +) + +func TestDevNothing(t *testing.T) { + if !*devFlag { + t.Log("not enabled") + return + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/file.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/file.go new file mode 100644 index 00000000000..94feda5efb0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/file.go @@ -0,0 +1,50 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "os" +) + +// FileAccessor is the concrete type returned by NewFile and OpenFile. +type FileAccessor struct { + *os.File +} + +// Implementation of Accessor. +func (f *FileAccessor) BeginUpdate() error { return nil } + +// Implementation of Accessor. +func (f *FileAccessor) EndUpdate() error { return nil } + +// NewFile returns an Accessor backed by an os.File named name, It opens the +// named file with specified flag (os.O_RDWR etc.) and perm, (0666 etc.) if +// applicable. If successful, methods on the returned Accessor can be used for +// I/O. It returns the Accessor and an Error, if any. +// +// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op. +func NewFile(name string, flag int, perm os.FileMode) (store Accessor, err error) { + var f FileAccessor + if f.File, err = os.OpenFile(name, flag, perm); err == nil { + store = &f + } + return +} + +// OpenFile returns an Accessor backed by an existing os.File named name, It +// opens the named file with specified flag (os.O_RDWR etc.) and perm, (0666 +// etc.) if applicable. If successful, methods on the returned Accessor can be +// used for I/O. It returns the Accessor and an Error, if any. +// +// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op. +func OpenFile(name string, flag int, perm os.FileMode) (store Accessor, err error) { + var f FileAccessor + if f.File, err = os.OpenFile(name, flag, perm); err == nil { + store = &f + } + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem.go new file mode 100644 index 00000000000..7cda0b667ef --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem.go @@ -0,0 +1,161 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "errors" + "fmt" + "io/ioutil" + "math" + "os" +) + +//TODO -> exported type w/ exported fields +type memaccessor struct { + f *os.File + fi *FileInfo + b []byte +} + +// Implementation of Accessor. +func (m *memaccessor) BeginUpdate() error { return nil } + +// Implementation of Accessor. +func (f *memaccessor) EndUpdate() error { return nil } + +// NewMem returns a new Accessor backed by an os.File. The returned Accessor +// keeps all of the store content in memory. The memory and file images are +// synced only by Sync and Close. Recomended for small amounts of data only +// and content which may be lost on process kill/crash. NewMem return the +// Accessor or an error of any. +// +// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op. +func NewMem(f *os.File) (store Accessor, err error) { + a := &memaccessor{f: f} + if err = f.Truncate(0); err != nil { + return + } + + var fi os.FileInfo + if fi, err = a.f.Stat(); err != nil { + return + } + + a.fi = NewFileInfo(fi, a) + store = a + return +} + +// OpenMem return a new Accessor backed by an os.File. The store content is +// loaded from f. The returned Accessor keeps all of the store content in +// memory. The memory and file images are synced only Sync and Close. +// Recomended for small amounts of data only and content which may be lost on +// process kill/crash. OpenMem return the Accessor or an error of any. +// +// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op. +func OpenMem(f *os.File) (store Accessor, err error) { + a := &memaccessor{f: f} + if a.b, err = ioutil.ReadAll(a.f); err != nil { + a.f.Close() + return + } + + var fi os.FileInfo + if fi, err = a.f.Stat(); err != nil { + a.f.Close() + return + } + + a.fi = NewFileInfo(fi, a) + store = a + return +} + +// Close implements Accessor. Specifically it synchronizes the memory and file images. +func (a *memaccessor) Close() (err error) { + defer func() { + a.b = nil + if a.f != nil { + if e := a.f.Close(); e != nil && err == nil { + err = e + } + } + a.f = nil + }() + + return a.Sync() +} + +func (a *memaccessor) Name() string { + return a.f.Name() +} + +func (a *memaccessor) ReadAt(b []byte, off int64) (n int, err error) { + if off < 0 || off > math.MaxInt32 { + return -1, fmt.Errorf("ReadAt: illegal offset %#x", off) + } + + rq, fp := len(b), int(off) + if fp+rq > len(a.b) { + return -1, fmt.Errorf("ReadAt: illegal rq %#x @ offset %#x, len %#x", rq, fp, len(a.b)) + } + + copy(b, a.b[fp:]) + return +} + +func (a *memaccessor) Stat() (fi os.FileInfo, err error) { + i := a.fi + i.FSize = int64(len(a.b)) + fi = i + return +} + +// Sync implements Accessor. Specifically it synchronizes the memory and file images. +func (a *memaccessor) Sync() (err error) { + var n int + if n, err = a.f.WriteAt(a.b, 0); n != len(a.b) { + return + } + + return a.f.Truncate(int64(len(a.b))) +} + +func (a *memaccessor) Truncate(size int64) (err error) { + defer func() { + if e := recover(); e != nil { + err = e.(error) + } + }() + + if size > math.MaxInt32 { + panic(errors.New("truncate: illegal size")) + } + + a.b = a.b[:int(size)] + return +} + +func (a *memaccessor) WriteAt(b []byte, off int64) (n int, err error) { + if off < 0 || off > math.MaxInt32 { + return -1, errors.New("WriteAt: illegal offset") + } + + rq, fp, size := len(b), int(off), len(a.b) + if need := rq + fp; need > size { + if need <= cap(a.b) { + a.b = a.b[:need] + } else { + nb := make([]byte, need, 2*need) + copy(nb, a.b) + a.b = nb + } + } + + copy(a.b[int(off):], b) + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem_test.go new file mode 100644 index 00000000000..921948c6ef7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem_test.go @@ -0,0 +1,15 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "testing" +) + +func Test(t *testing.T) { + t.Log("TODO placeholder") //TODO +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe.go new file mode 100644 index 00000000000..53b146a6eb5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe.go @@ -0,0 +1,74 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import "sync/atomic" + +// Probe collects usage statistics of the embeded Accessor. +// Probe itself IS an Accessor. +type Probe struct { + Accessor + Chain *Probe + OpsRd int64 + OpsWr int64 + BytesRd int64 + BytesWr int64 + SectorsRd int64 // Assuming 512 byte sector size + SectorsWr int64 +} + +// NewProbe returns a newly created probe which embedes the src Accessor. +// The retuned *Probe satisfies Accessor. if chain != nil then Reset() +// is cascaded down the chained Probes. +func NewProbe(src Accessor, chain *Probe) *Probe { + return &Probe{Accessor: src, Chain: chain} +} + +func reset(n *int64) { + atomic.AddInt64(n, -atomic.AddInt64(n, 0)) +} + +// Reset zeroes the collected statistics of p. +func (p *Probe) Reset() { + if p.Chain != nil { + p.Chain.Reset() + } + reset(&p.OpsRd) + reset(&p.OpsWr) + reset(&p.BytesRd) + reset(&p.BytesWr) + reset(&p.SectorsRd) + reset(&p.SectorsWr) +} + +func (p *Probe) ReadAt(b []byte, off int64) (n int, err error) { + n, err = p.Accessor.ReadAt(b, off) + atomic.AddInt64(&p.OpsRd, 1) + atomic.AddInt64(&p.BytesRd, int64(n)) + if n <= 0 { + return + } + + sectorFirst := off >> 9 + sectorLast := (off + int64(n) - 1) >> 9 + atomic.AddInt64(&p.SectorsRd, sectorLast-sectorFirst+1) + return +} + +func (p *Probe) WriteAt(b []byte, off int64) (n int, err error) { + n, err = p.Accessor.WriteAt(b, off) + atomic.AddInt64(&p.OpsWr, 1) + atomic.AddInt64(&p.BytesWr, int64(n)) + if n <= 0 { + return + } + + sectorFirst := off >> 9 + sectorLast := (off + int64(n) - 1) >> 9 + atomic.AddInt64(&p.SectorsWr, sectorLast-sectorFirst+1) + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe_test.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe_test.go new file mode 100644 index 00000000000..00eca8ee554 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe_test.go @@ -0,0 +1,86 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +import ( + "os" + "testing" +) + +func (p *Probe) assert(t *testing.T, msg int, opsRd, opsWr, bytesRd, bytesWr, sectorsRd, sectorsWr int64) { + if n := p.OpsRd; n != opsRd { + t.Fatal(msg, n, opsRd) + } + + if n := p.OpsWr; n != opsWr { + t.Fatal(msg+1, n, opsWr) + } + + if n := p.BytesRd; n != bytesRd { + t.Fatal(msg+2, n, bytesRd) + } + + if n := p.BytesWr; n != bytesWr { + t.Fatal(msg+3, n, bytesWr) + } + + if n := p.SectorsRd; n != sectorsRd { + t.Fatal(msg+4, n, sectorsRd) + } + + if n := p.SectorsWr; n != sectorsWr { + t.Fatal(msg+5, n, sectorsWr) + } +} + +func TestProbe(t *testing.T) { + return //TODO disabled due to atomic.AddInt64 failing on W32 + const fn = "test.tmp" + + store, err := NewFile(fn, os.O_CREATE|os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + t.Fatal(10, err) + } + + defer func() { + ec := store.Close() + er := os.Remove(fn) + if ec != nil { + t.Fatal(10000, ec) + } + if er != nil { + t.Fatal(10001, er) + } + }() + + probe := NewProbe(store, nil) + if n, err := probe.WriteAt([]byte{1}, 0); n != 1 { + t.Fatal(20, err) + } + + probe.assert(t, 30, 0, 1, 0, 1, 0, 1) + b := []byte{0} + if n, err := probe.ReadAt(b, 0); n != 1 { + t.Fatal(40, err) + } + + if n := b[0]; n != 1 { + t.Fatal(50, n, 1) + } + + probe.assert(t, 60, 1, 1, 1, 1, 1, 1) + if n, err := probe.WriteAt([]byte{2, 3}, 510); n != 2 { + t.Fatal(70, err) + } + + probe.assert(t, 80, 1, 2, 1, 3, 1, 2) + if n, err := probe.WriteAt([]byte{2, 3}, 511); n != 2 { + t.Fatal(90, err) + } + + probe.assert(t, 100, 1, 3, 1, 5, 1, 4) +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/storage.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/storage.go new file mode 100644 index 00000000000..4956053a0c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/storage.go @@ -0,0 +1,141 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +// WIP: Package storage defines and implements storage providers and store accessors. +package storage + +import ( + "os" + "sync" + "time" +) + +// FileInfo is a type implementing os.FileInfo which has setable fields, like +// the older os.FileInfo used to have. It is used wehere e.g. the Size is +// needed to be faked (encapsulated/memory only file, file cache, etc.). +type FileInfo struct { + FName string // base name of the file + FSize int64 // length in bytes + FMode os.FileMode // file mode bits + FModTime time.Time // modification time + FIsDir bool // abbreviation for Mode().IsDir() + sys interface{} // underlying data source (can be nil) +} + +// NewFileInfo creates FileInfo from os.FileInfo fi. +func NewFileInfo(fi os.FileInfo, sys interface{}) *FileInfo { + return &FileInfo{fi.Name(), fi.Size(), fi.Mode(), fi.ModTime(), fi.IsDir(), sys} +} + +// Implementation of os.FileInfo +func (fi *FileInfo) Name() string { + return fi.FName +} + +// Implementation of os.FileInfo +func (fi *FileInfo) Size() int64 { + return fi.FSize +} + +// Implementation of os.FileInfo +func (fi *FileInfo) Mode() os.FileMode { + return fi.FMode +} + +// Implementation of os.FileInfo +func (fi *FileInfo) ModTime() time.Time { + return fi.FModTime +} + +// Implementation of os.FileInfo +func (fi *FileInfo) IsDir() bool { + return fi.FIsDir +} + +func (fi *FileInfo) Sys() interface{} { + return fi.sys +} + +// Accessor provides I/O methods to access a store. +type Accessor interface { + + // Close closes the store, rendering it unusable for I/O. It returns an + // error, if any. + Close() error + + // Name returns the name of the file as presented to Open. + Name() string + + // ReadAt reads len(b) bytes from the store starting at byte offset off. + // It returns the number of bytes read and the error, if any. + // EOF is signaled by a zero count with err set to os.EOF. + // ReadAt always returns a non-nil Error when n != len(b). + ReadAt(b []byte, off int64) (n int, err error) + + // Stat returns the FileInfo structure describing the store. It returns + // the os.FileInfo and an error, if any. + Stat() (fi os.FileInfo, err error) + + // Sync commits the current contents of the store to stable storage. + // Typically, this means flushing the file system's in-memory copy of + // recently written data to disk. + Sync() (err error) + + // Truncate changes the size of the store. It does not change the I/O + // offset. + Truncate(size int64) error + + // WriteAt writes len(b) bytes to the store starting at byte offset off. + // It returns the number of bytes written and an error, if any. + // WriteAt returns a non-nil Error when n != len(b). + WriteAt(b []byte, off int64) (n int, err error) + + // Before every [structural] change of a store the BeginUpdate is to be + // called and paired with EndUpdate after the change makes the store's + // state consistent again. Invocations of BeginUpdate may nest. On + // invoking the last non nested EndUpdate an implicit "commit" should + // be performed by the store/provider. The concrete mechanism is + // unspecified. It could be for example a write-ahead log. Stores may + // implement BeginUpdate and EndUpdate as a (documented) no op. + BeginUpdate() error + EndUpdate() error +} + +// Mutate is a helper/wrapper for executing f in between a.BeginUpdate and +// a.EndUpdate. Any parameters and/or return values except an error should be +// captured by a function literal passed as f. The returned err is either nil +// or the first non nil error returned from the sequence of execution: +// BeginUpdate, [f,] EndUpdate. The pair BeginUpdate/EndUpdate *is* invoked +// always regardles of any possible errors produced. Mutate doesn't handle +// panic, it should be used only with a function [literal] which doesn't panic. +// Otherwise the pairing of BeginUpdate/EndUpdate is not guaranteed. +// +// NOTE: If BeginUpdate, which is invoked before f, returns a non-nil error, +// then f is not invoked at all (but EndUpdate still is). +func Mutate(a Accessor, f func() error) (err error) { + defer func() { + if e := a.EndUpdate(); e != nil && err == nil { + err = e + } + }() + + if err = a.BeginUpdate(); err != nil { + return + } + + return f() +} + +// LockedMutate wraps Mutate in yet another layer consisting of a +// l.Lock/l.Unlock pair. All other limitations apply as in Mutate, e.g. no +// panics are allowed to happen - otherwise no guarantees can be made about +// Unlock matching the Lock. +func LockedMutate(a Accessor, l sync.Locker, f func() error) (err error) { + l.Lock() + defer l.Unlock() + + return Mutate(a, f) +} diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/storage/test_deps.go b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/test_deps.go new file mode 100644 index 00000000000..92ac44a5bf7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/storage/test_deps.go @@ -0,0 +1,13 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package storage + +// Pull test dependencies too. +// Enables easy 'go test X' after 'go get X' +import ( +// nothing yet +) diff --git a/Godeps/_workspace/src/github.com/cznic/fileutil/test_deps.go b/Godeps/_workspace/src/github.com/cznic/fileutil/test_deps.go new file mode 100644 index 00000000000..eec608ab3a6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/fileutil/test_deps.go @@ -0,0 +1,13 @@ +// Copyright (c) 2014 The fileutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +package fileutil + +// Pull test dependencies too. +// Enables easy 'go test X' after 'go get X' +import ( +// nothing yet +) diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/AUTHORS b/Godeps/_workspace/src/github.com/cznic/mathutil/AUTHORS new file mode 100644 index 00000000000..d04c4501186 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/AUTHORS @@ -0,0 +1,12 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +CZ.NIC z.s.p.o. +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/mathutil/CONTRIBUTORS new file mode 100644 index 00000000000..9c9a5dd84c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/CONTRIBUTORS @@ -0,0 +1,10 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Gary Burd +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/LICENSE b/Godeps/_workspace/src/github.com/cznic/mathutil/LICENSE new file mode 100644 index 00000000000..128a1b64a48 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The mathutil Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/Makefile b/Godeps/_workspace/src/github.com/cznic/mathutil/Makefile new file mode 100644 index 00000000000..b99ba9adf2f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/Makefile @@ -0,0 +1,31 @@ +# Copyright (c) 2014 The mathutil Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +.PHONY: all todo clean nuke + +grep=--include=*.go --include=*.run --include=*.y + +all: editor + go build + go vet || true + golint . + go install + make todo + +clean: + go clean + +editor: + go fmt + go test -i + go test + +todo: + @grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * || true + @grep -nr $(grep) TODO * || true + @grep -nr $(grep) BUG * || true + @grep -nr $(grep) println * || true + +nuke: clean + go clean -i diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/README b/Godeps/_workspace/src/github.com/cznic/mathutil/README new file mode 100644 index 00000000000..a9ee59c401c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/README @@ -0,0 +1,10 @@ +This is a goinstall-able mirror of modified code already published at: +http://git.nic.cz/redmine/projects/gornd/repository + +Packages in this repository: + +Install: $ go get github.com/cznic/mathutil +Godocs: http://godoc.org/github.com/cznic/mathutil + +Install: $ go get github.com/cznic/mathutil/mersenne +Godocs: http://godoc.org/github.com/cznic/mathutil/mersenne diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/all_test.go b/Godeps/_workspace/src/github.com/cznic/mathutil/all_test.go new file mode 100644 index 00000000000..38427ad4ffb --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/all_test.go @@ -0,0 +1,3530 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +import ( + "fmt" + "math" + "math/big" + "math/rand" + "os" + "path" + "runtime" + "sort" + "strings" + "sync" + "testing" +) + +func caller(s string, va ...interface{}) { + _, fn, fl, _ := runtime.Caller(2) + fmt.Fprintf(os.Stderr, "caller: %s:%d: ", path.Base(fn), fl) + fmt.Fprintf(os.Stderr, s, va...) + fmt.Fprintln(os.Stderr) + _, fn, fl, _ = runtime.Caller(1) + fmt.Fprintf(os.Stderr, "\tcallee: %s:%d: ", path.Base(fn), fl) + fmt.Fprintln(os.Stderr) +} + +func dbg(s string, va ...interface{}) { + if s == "" { + s = strings.Repeat("%v ", len(va)) + } + _, fn, fl, _ := runtime.Caller(1) + fmt.Fprintf(os.Stderr, "dbg %s:%d: ", path.Base(fn), fl) + fmt.Fprintf(os.Stderr, s, va...) + fmt.Fprintln(os.Stderr) +} + +func TODO(...interface{}) string { + _, fn, fl, _ := runtime.Caller(1) + return fmt.Sprintf("TODO: %s:%d:\n", path.Base(fn), fl) +} + +func use(...interface{}) {} + +// ============================================================================ + +func r32() *FC32 { + r, err := NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + panic(err) + } + + return r +} + +var ( + r64lo = big.NewInt(math.MinInt64) + r64hi = big.NewInt(math.MaxInt64) + _3 = big.NewInt(3) + MinIntM1 = MinInt + MaxIntP1 = MaxInt + MaxUintP1 uint = MaxUint +) + +func init() { + MinIntM1-- + MaxIntP1++ + MaxUintP1++ +} + +func r64() *FCBig { + r, err := NewFCBig(r64lo, r64hi, true) + if err != nil { + panic(err) + } + + return r +} + +func benchmark1eN(b *testing.B, r *FC32) { + b.StartTimer() + for i := 0; i < b.N; i++ { + r.Next() + } +} + +func BenchmarkFC1e3(b *testing.B) { + b.StopTimer() + r, _ := NewFC32(0, 1e3, false) + benchmark1eN(b, r) +} + +func BenchmarkFC1e6(b *testing.B) { + b.StopTimer() + r, _ := NewFC32(0, 1e6, false) + benchmark1eN(b, r) +} + +func BenchmarkFC1e9(b *testing.B) { + b.StopTimer() + r, _ := NewFC32(0, 1e9, false) + benchmark1eN(b, r) +} + +func Test0(t *testing.T) { + const N = 10000 + for n := 1; n < N; n++ { + lo, hi := 0, n-1 + period := int64(hi) - int64(lo) + 1 + r, err := NewFC32(lo, hi, false) + if err != nil { + t.Fatal(err) + } + if r.Cycle()-period > period { + t.Fatalf("Cycle exceeds 2 * period") + } + } + for n := 1; n < N; n++ { + lo, hi := 0, n-1 + period := int64(hi) - int64(lo) + 1 + r, err := NewFC32(lo, hi, true) + if err != nil { + t.Fatal(err) + } + if r.Cycle()-2*period > period { + t.Fatalf("Cycle exceeds 3 * period") + } + } +} + +func Test1(t *testing.T) { + const ( + N = 360 + S = 3 + ) + for hq := 0; hq <= 1; hq++ { + for n := 1; n < N; n++ { + for seed := 0; seed < S; seed++ { + lo, hi := -n, 2*n + period := int64(hi - lo + 1) + r, err := NewFC32(lo, hi, hq == 1) + if err != nil { + t.Fatal(err) + } + r.Seed(int64(seed)) + m := map[int]bool{} + v := make([]int, period, period) + p := make([]int64, period, period) + for i := lo; i <= hi; i++ { + x := r.Next() + p[i-lo] = r.Pos() + if x < lo || x > hi { + t.Fatal("t1.0") + } + if m[x] { + t.Fatal("t1.1") + } + m[x] = true + v[i-lo] = x + } + for i := lo; i <= hi; i++ { + x := r.Next() + if x < lo || x > hi { + t.Fatal("t1.2") + } + if !m[x] { + t.Fatal("t1.3") + } + if x != v[i-lo] { + t.Fatal("t1.4") + } + if r.Pos() != p[i-lo] { + t.Fatal("t1.5") + } + m[x] = false + } + for i := lo; i <= hi; i++ { + r.Seek(p[i-lo] + 1) + x := r.Prev() + if x < lo || x > hi { + t.Fatal("t1.6") + } + if x != v[i-lo] { + t.Fatal("t1.7") + } + } + } + } + } +} + +func Test2(t *testing.T) { + const ( + N = 370 + S = 3 + ) + for hq := 0; hq <= 1; hq++ { + for n := 1; n < N; n++ { + for seed := 0; seed < S; seed++ { + lo, hi := -n, 2*n + period := int64(hi - lo + 1) + r, err := NewFC32(lo, hi, hq == 1) + if err != nil { + t.Fatal(err) + } + r.Seed(int64(seed)) + m := map[int]bool{} + v := make([]int, period, period) + p := make([]int64, period, period) + for i := lo; i <= hi; i++ { + x := r.Prev() + p[i-lo] = r.Pos() + if x < lo || x > hi { + t.Fatal("t2.0") + } + if m[x] { + t.Fatal("t2.1") + } + m[x] = true + v[i-lo] = x + } + for i := lo; i <= hi; i++ { + x := r.Prev() + if x < lo || x > hi { + t.Fatal("t2.2") + } + if !m[x] { + t.Fatal("t2.3") + } + if x != v[i-lo] { + t.Fatal("t2.4") + } + if r.Pos() != p[i-lo] { + t.Fatal("t2.5") + } + m[x] = false + } + for i := lo; i <= hi; i++ { + s := p[i-lo] - 1 + if s < 0 { + s = r.Cycle() - 1 + } + r.Seek(s) + x := r.Next() + if x < lo || x > hi { + t.Fatal("t2.6") + } + if x != v[i-lo] { + t.Fatal("t2.7") + } + } + } + } + } +} + +func benchmarkBig1eN(b *testing.B, r *FCBig) { + b.StartTimer() + for i := 0; i < b.N; i++ { + r.Next() + } +} + +func BenchmarkFCBig1e3(b *testing.B) { + b.StopTimer() + hi := big.NewInt(0).SetInt64(1e3) + r, _ := NewFCBig(big0, hi, false) + benchmarkBig1eN(b, r) +} + +func BenchmarkFCBig1e6(b *testing.B) { + b.StopTimer() + hi := big.NewInt(0).SetInt64(1e6) + r, _ := NewFCBig(big0, hi, false) + benchmarkBig1eN(b, r) +} + +func BenchmarkFCBig1e9(b *testing.B) { + b.StopTimer() + hi := big.NewInt(0).SetInt64(1e9) + r, _ := NewFCBig(big0, hi, false) + benchmarkBig1eN(b, r) +} + +func BenchmarkFCBig1e12(b *testing.B) { + b.StopTimer() + hi := big.NewInt(0).SetInt64(1e12) + r, _ := NewFCBig(big0, hi, false) + benchmarkBig1eN(b, r) +} + +func BenchmarkFCBig1e15(b *testing.B) { + b.StopTimer() + hi := big.NewInt(0).SetInt64(1e15) + r, _ := NewFCBig(big0, hi, false) + benchmarkBig1eN(b, r) +} + +func BenchmarkFCBig1e18(b *testing.B) { + b.StopTimer() + hi := big.NewInt(0).SetInt64(1e18) + r, _ := NewFCBig(big0, hi, false) + benchmarkBig1eN(b, r) +} + +var ( + big0 = big.NewInt(0) +) + +func TestBig0(t *testing.T) { + const N = 7400 + lo := big.NewInt(0) + hi := big.NewInt(0) + period := big.NewInt(0) + c := big.NewInt(0) + for n := int64(1); n < N; n++ { + hi.SetInt64(n - 1) + period.Set(hi) + period.Sub(period, lo) + period.Add(period, _1) + r, err := NewFCBig(lo, hi, false) + if err != nil { + t.Fatal(err) + } + if r.cycle.Cmp(period) < 0 { + t.Fatalf("Period exceeds cycle") + } + c.Set(r.Cycle()) + c.Sub(c, period) + if c.Cmp(period) > 0 { + t.Fatalf("Cycle exceeds 2 * period") + } + } + for n := int64(1); n < N; n++ { + hi.SetInt64(n - 1) + period.Set(hi) + period.Sub(period, lo) + period.Add(period, _1) + r, err := NewFCBig(lo, hi, true) + if err != nil { + t.Fatal(err) + } + if r.cycle.Cmp(period) < 0 { + t.Fatalf("Period exceeds cycle") + } + c.Set(r.Cycle()) + c.Sub(c, period) + c.Sub(c, period) + if c.Cmp(period) > 0 { + t.Fatalf("Cycle exceeds 3 * period") + } + } +} + +func TestBig1(t *testing.T) { + const ( + N = 120 + S = 3 + ) + lo := big.NewInt(0) + hi := big.NewInt(0) + seek := big.NewInt(0) + for hq := 0; hq <= 1; hq++ { + for n := int64(1); n < N; n++ { + for seed := 0; seed < S; seed++ { + lo64 := -n + hi64 := 2 * n + lo.SetInt64(lo64) + hi.SetInt64(hi64) + period := hi64 - lo64 + 1 + r, err := NewFCBig(lo, hi, hq == 1) + if err != nil { + t.Fatal(err) + } + r.Seed(int64(seed)) + m := map[int64]bool{} + v := make([]int64, period, period) + p := make([]int64, period, period) + for i := lo64; i <= hi64; i++ { + x := r.Next().Int64() + p[i-lo64] = r.Pos().Int64() + if x < lo64 || x > hi64 { + t.Fatal("tb1.0") + } + if m[x] { + t.Fatal("tb1.1") + } + m[x] = true + v[i-lo64] = x + } + for i := lo64; i <= hi64; i++ { + x := r.Next().Int64() + if x < lo64 || x > hi64 { + t.Fatal("tb1.2") + } + if !m[x] { + t.Fatal("tb1.3") + } + if x != v[i-lo64] { + t.Fatal("tb1.4") + } + if r.Pos().Int64() != p[i-lo64] { + t.Fatal("tb1.5") + } + m[x] = false + } + for i := lo64; i <= hi64; i++ { + r.Seek(seek.SetInt64(p[i-lo64] + 1)) + x := r.Prev().Int64() + if x < lo64 || x > hi64 { + t.Fatal("tb1.6") + } + if x != v[i-lo64] { + t.Fatal("tb1.7") + } + } + } + } + } +} + +func TestBig2(t *testing.T) { + const ( + N = 120 + S = 3 + ) + lo := big.NewInt(0) + hi := big.NewInt(0) + seek := big.NewInt(0) + for hq := 0; hq <= 1; hq++ { + for n := int64(1); n < N; n++ { + for seed := 0; seed < S; seed++ { + lo64, hi64 := -n, 2*n + lo.SetInt64(lo64) + hi.SetInt64(hi64) + period := hi64 - lo64 + 1 + r, err := NewFCBig(lo, hi, hq == 1) + if err != nil { + t.Fatal(err) + } + r.Seed(int64(seed)) + m := map[int64]bool{} + v := make([]int64, period, period) + p := make([]int64, period, period) + for i := lo64; i <= hi64; i++ { + x := r.Prev().Int64() + p[i-lo64] = r.Pos().Int64() + if x < lo64 || x > hi64 { + t.Fatal("tb2.0") + } + if m[x] { + t.Fatal("tb2.1") + } + m[x] = true + v[i-lo64] = x + } + for i := lo64; i <= hi64; i++ { + x := r.Prev().Int64() + if x < lo64 || x > hi64 { + t.Fatal("tb2.2") + } + if !m[x] { + t.Fatal("tb2.3") + } + if x != v[i-lo64] { + t.Fatal("tb2.4") + } + if r.Pos().Int64() != p[i-lo64] { + t.Fatal("tb2.5") + } + m[x] = false + } + for i := lo64; i <= hi64; i++ { + s := p[i-lo64] - 1 + if s < 0 { + s = r.Cycle().Int64() - 1 + } + r.Seek(seek.SetInt64(s)) + x := r.Next().Int64() + if x < lo64 || x > hi64 { + t.Fatal("tb2.6") + } + if x != v[i-lo64] { + t.Fatal("tb2.7") + } + } + } + } + } +} + +func TestPermutations(t *testing.T) { + data := sort.IntSlice{3, 2, 1} + check := [][]int{ + {1, 2, 3}, + {1, 3, 2}, + {2, 1, 3}, + {2, 3, 1}, + {3, 1, 2}, + {3, 2, 1}, + } + i := 0 + for PermutationFirst(data); ; i++ { + if i >= len(check) { + t.Fatalf("too much permutations generated: %d > %d", i+1, len(check)) + } + + for j, v := range check[i] { + got := data[j] + if got != v { + t.Fatalf("permutation %d:\ndata: %v\ncheck: %v\nexpected data[%d] == %d, got %d", i, data, check[i], j, v, got) + } + } + + if !PermutationNext(data) { + if i != len(check)-1 { + t.Fatal("permutations generated", i, "expected", len(check)) + } + break + } + } +} + +func TestIsPrime(t *testing.T) { + const p4M = 283146 // # of primes < 4e6 + n := 0 + for i := uint32(0); i <= 4e6; i++ { + if IsPrime(i) { + n++ + } + } + t.Log(n) + if n != p4M { + t.Fatal(n) + } +} + +func BenchmarkIsPrime(b *testing.B) { + b.StopTimer() + n := make([]uint32, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = rng.Uint32() + } + b.StartTimer() + for _, n := range n { + IsPrime(n) + } +} + +func BenchmarkNextPrime(b *testing.B) { + b.StopTimer() + n := make([]uint32, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = rng.Uint32() + } + b.StartTimer() + for _, n := range n { + NextPrime(n) + } +} + +func BenchmarkIsPrimeUint64(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + a := make([]uint64, N) + r := r64() + for i := range a { + a[i] = uint64(r.Next().Int64()) + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + IsPrimeUint64(a[i&(N-1)]) + } +} + +func BenchmarkNextPrimeUint64(b *testing.B) { + b.StopTimer() + n := make([]uint64, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = uint64(rng.Int63()) + if i&1 == 0 { + n[i] ^= 1 << 63 + } + } + b.StartTimer() + for _, n := range n { + NextPrimeUint64(n) + } +} + +func TestNextPrime(t *testing.T) { + const p4M = 283146 // # of primes < 4e6 + n := 0 + var p uint32 + for { + p, _ = NextPrime(p) + if p >= 4e6 { + break + } + n++ + } + t.Log(n) + if n != p4M { + t.Fatal(n) + } +} + +func TestNextPrime2(t *testing.T) { + type data struct { + x uint32 + y uint32 + ok bool + } + tests := []data{ + {0, 2, true}, + {1, 2, true}, + {2, 3, true}, + {3, 5, true}, + {math.MaxUint32, 0, false}, + {math.MaxUint32 - 1, 0, false}, + {math.MaxUint32 - 2, 0, false}, + {math.MaxUint32 - 3, 0, false}, + {math.MaxUint32 - 4, 0, false}, + {math.MaxUint32 - 5, math.MaxUint32 - 4, true}, + } + + for _, test := range tests { + y, ok := NextPrime(test.x) + if ok != test.ok || ok && y != test.y { + t.Fatalf("x %d, got y %d ok %t, expected y %d ok %t", test.x, y, ok, test.y, test.ok) + } + } +} + +func TestNextPrimeUint64(t *testing.T) { + const ( + lo = 2000000000000000000 + hi = 2000000000000100000 + k = 2346 // PrimePi(hi)-PrimePi(lo) + ) + n := 0 + p := uint64(lo) - 1 + var ok bool + for { + p0 := p + p, ok = NextPrimeUint64(p) + if !ok { + t.Fatal(p0) + } + + if p > hi { + break + } + + n++ + } + if n != k { + t.Fatal(n, k) + } +} + +func TestISqrt(t *testing.T) { + for n := int64(0); n < 5e6; n++ { + x := int64(ISqrt(uint32(n))) + if x2 := x * x; x2 > n { + t.Fatalf("got ISqrt(%d) == %d, too big", n, x) + } + if x2 := x*x + 2*x + 1; x2 < n { + t.Fatalf("got ISqrt(%d) == %d, too low", n, x) + } + } + for n := int64(math.MaxUint32); n > math.MaxUint32-5e6; n-- { + x := int64(ISqrt(uint32(n))) + if x2 := x * x; x2 > n { + t.Fatalf("got ISqrt(%d) == %d, too big", n, x) + } + if x2 := x*x + 2*x + 1; x2 < n { + t.Fatalf("got ISqrt(%d) == %d, too low", n, x) + } + } + rng := rand.New(rand.NewSource(1)) + for i := 0; i < 5e6; i++ { + n := int64(rng.Uint32()) + x := int64(ISqrt(uint32(n))) + if x2 := x * x; x2 > n { + t.Fatalf("got ISqrt(%d) == %d, too big", n, x) + } + if x2 := x*x + 2*x + 1; x2 < n { + t.Fatalf("got ISqrt(%d) == %d, too low", n, x) + } + } +} + +func TestSqrtUint64(t *testing.T) { + for n := uint64(0); n < 2e6; n++ { + x := SqrtUint64(n) + if x > math.MaxUint32 { + t.Fatalf("got Sqrt(%d) == %d, too big", n, x) + } + if x2 := x * x; x2 > n { + t.Fatalf("got Sqrt(%d) == %d, too big", n, x) + } + if x2 := x*x + 2*x + 1; x2 < n { + t.Fatalf("got Sqrt(%d) == %d, too low", n, x) + } + } + const H = uint64(18446744056529682436) + for n := H; n > H-2e6; n-- { + x := SqrtUint64(n) + if x > math.MaxUint32 { + t.Fatalf("got Sqrt(%d) == %d, too big", n, x) + } + if x2 := x * x; x2 > n { + t.Fatalf("got Sqrt(%d) == %d, too big", n, x) + } + if x2 := x*x + 2*x + 1; x2 < n { + t.Fatalf("got Sqrt(%d) == %d, too low", n, x) + } + } + rng := rand.New(rand.NewSource(1)) + for i := 0; i < 2e6; i++ { + n := uint64(rng.Uint32())<<31 | uint64(rng.Uint32()) + x := SqrtUint64(n) + if x2 := x * x; x2 > n { + t.Fatalf("got Sqrt(%d) == %d, too big", n, x) + } + if x2 := x*x + 2*x + 1; x2 < n { + t.Fatalf("got Sqrt(%d) == %d, too low", n, x) + } + } +} + +func TestSqrtBig(t *testing.T) { + const N = 3e4 + var n, lim, x2 big.Int + lim.SetInt64(N) + for n.Cmp(&lim) != 0 { + x := SqrtBig(&n) + x2.Mul(x, x) + if x.Cmp(&n) > 0 { + t.Fatalf("got sqrt(%s) == %s, too big", &n, x) + } + x2.Add(&x2, x) + x2.Add(&x2, x) + x2.Add(&x2, _1) + if x2.Cmp(&n) < 0 { + t.Fatalf("got sqrt(%s) == %s, too low", &n, x) + } + n.Add(&n, _1) + } + rng := rand.New(rand.NewSource(1)) + var h big.Int + h.SetBit(&h, 1e3, 1) + for i := 0; i < N; i++ { + n.Rand(rng, &h) + x := SqrtBig(&n) + x2.Mul(x, x) + if x.Cmp(&n) > 0 { + t.Fatalf("got sqrt(%s) == %s, too big", &n, x) + } + x2.Add(&x2, x) + x2.Add(&x2, x) + x2.Add(&x2, _1) + if x2.Cmp(&n) < 0 { + t.Fatalf("got sqrt(%s) == %s, too low", &n, x) + } + } +} + +func TestFactorInt(t *testing.T) { + chk := func(n uint64, f []FactorTerm) bool { + if n < 2 { + return len(f) == 0 + } + + for i := 1; i < len(f); i++ { // verify ordering + if t, u := f[i-1], f[i]; t.Prime >= u.Prime { + return false + } + } + + x := uint64(1) + for _, v := range f { + if p := v.Prime; p < 0 || !IsPrime(uint32(v.Prime)) { + return false + } + + for i := uint32(0); i < v.Power; i++ { + x *= uint64(v.Prime) + if x > math.MaxUint32 { + return false + } + } + } + return x == n + } + + for n := uint64(0); n < 3e5; n++ { + f := FactorInt(uint32(n)) + if !chk(n, f) { + t.Fatalf("bad FactorInt(%d): %v", n, f) + } + } + for n := uint64(math.MaxUint32); n > math.MaxUint32-12e4; n-- { + f := FactorInt(uint32(n)) + if !chk(n, f) { + t.Fatalf("bad FactorInt(%d): %v", n, f) + } + } + rng := rand.New(rand.NewSource(1)) + for i := 0; i < 13e4; i++ { + n := rng.Uint32() + f := FactorInt(n) + if !chk(uint64(n), f) { + t.Fatalf("bad FactorInt(%d): %v", n, f) + } + } +} + +func TestFactorIntB(t *testing.T) { + const N = 3e5 // must be < math.MaxInt32 + factors := make([][]FactorTerm, N+1) + // set up the divisors + for prime := uint32(2); prime <= N; prime, _ = NextPrime(prime) { + for n := int(prime); n <= N; n += int(prime) { + factors[n] = append(factors[n], FactorTerm{prime, 0}) + } + } + // set up the powers + for n := 2; n <= N; n++ { + f := factors[n] + m := uint32(n) + for i, v := range f { + for m%v.Prime == 0 { + m /= v.Prime + v.Power++ + } + f[i] = v + } + factors[n] = f + } + // check equal + for n, e := range factors { + g := FactorInt(uint32(n)) + if len(e) != len(g) { + t.Fatal(n, "len", g, "!=", e) + } + + for i, ev := range e { + gv := g[i] + if ev.Prime != gv.Prime { + t.Fatal(n, "prime", gv, ev) + } + + if ev.Power != gv.Power { + t.Fatal(n, "power", gv, ev) + } + } + } +} + +func BenchmarkISqrt(b *testing.B) { + b.StopTimer() + n := make([]uint32, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = rng.Uint32() + } + b.StartTimer() + for _, n := range n { + ISqrt(n) + } +} + +func BenchmarkSqrtUint64(b *testing.B) { + b.StopTimer() + n := make([]uint64, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = uint64(rng.Uint32())<<32 | uint64(rng.Uint32()) + } + b.StartTimer() + for _, n := range n { + SqrtUint64(n) + } +} + +func benchmarkSqrtBig(b *testing.B, bits int) { + b.StopTimer() + n := make([]*big.Int, b.N) + rng := rand.New(rand.NewSource(1)) + var nn, h big.Int + h.SetBit(&h, bits, 1) + for i := 0; i < b.N; i++ { + n[i] = nn.Rand(rng, &h) + } + runtime.GC() + b.StartTimer() + for _, n := range n { + SqrtBig(n) + } +} + +func BenchmarkSqrtBig2e1e1(b *testing.B) { + benchmarkSqrtBig(b, 1e1) +} + +func BenchmarkSqrtBig2e1e2(b *testing.B) { + benchmarkSqrtBig(b, 1e2) +} + +func BenchmarkSqrtBig2e1e3(b *testing.B) { + benchmarkSqrtBig(b, 1e3) +} + +func BenchmarkSqrtBig2e1e4(b *testing.B) { + benchmarkSqrtBig(b, 1e4) +} + +func BenchmarkSqrtBig2e1e5(b *testing.B) { + benchmarkSqrtBig(b, 1e5) +} + +func BenchmarkFactorInt(b *testing.B) { + b.StopTimer() + n := make([]uint32, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = rng.Uint32() + } + b.StartTimer() + for _, n := range n { + FactorInt(n) + } +} + +func TestIsPrimeUint16(t *testing.T) { + for n := 0; n <= math.MaxUint16; n++ { + if IsPrimeUint16(uint16(n)) != IsPrime(uint32(n)) { + t.Fatal(n) + } + } +} + +func BenchmarkIsPrimeUint16(b *testing.B) { + b.StopTimer() + n := make([]uint16, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = uint16(rng.Uint32()) + } + b.StartTimer() + for _, n := range n { + IsPrimeUint16(n) + } +} + +func TestNextPrimeUint16(t *testing.T) { + for n := 0; n <= math.MaxUint16; n++ { + p, ok := NextPrimeUint16(uint16(n)) + p2, ok2 := NextPrime(uint32(n)) + switch { + case ok: + if !ok2 || uint32(p) != p2 { + t.Fatal(n, p, ok) + } + case !ok && ok2: + if p2 < 65536 { + t.Fatal(n, p, ok) + } + } + } +} + +func BenchmarkNextPrimeUint16(b *testing.B) { + b.StopTimer() + n := make([]uint16, b.N) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < b.N; i++ { + n[i] = uint16(rng.Uint32()) + } + b.StartTimer() + for _, n := range n { + NextPrimeUint16(n) + } +} + +/* + +From: http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetKernighan + +Counting bits set, Brian Kernighan's way + +unsigned int v; // count the number of bits set in v +unsigned int c; // c accumulates the total bits set in v +for (c = 0; v; c++) +{ + v &= v - 1; // clear the least significant bit set +} + +Brian Kernighan's method goes through as many iterations as there are set bits. +So if we have a 32-bit word with only the high bit set, then it will only go +once through the loop. + +Published in 1988, the C Programming Language 2nd Ed. (by Brian W. Kernighan +and Dennis M. Ritchie) mentions this in exercise 2-9. On April 19, 2006 Don +Knuth pointed out to me that this method "was first published by Peter Wegner +in CACM 3 (1960), 322. (Also discovered independently by Derrick Lehmer and +published in 1964 in a book edited by Beckenbach.)" +*/ +func bcnt(v uint64) (c int) { + for ; v != 0; c++ { + v &= v - 1 + } + return +} + +func TestPopCount(t *testing.T) { + const N = 4e5 + maxUint64 := big.NewInt(0) + maxUint64.SetBit(maxUint64, 64, 1) + maxUint64.Sub(maxUint64, big.NewInt(1)) + rng := r64() + for i := 0; i < N; i++ { + n := uint64(rng.Next().Int64()) + if g, e := PopCountByte(byte(n)), bcnt(uint64(byte(n))); g != e { + t.Fatal(n, g, e) + } + + if g, e := PopCountUint16(uint16(n)), bcnt(uint64(uint16(n))); g != e { + t.Fatal(n, g, e) + } + + if g, e := PopCountUint32(uint32(n)), bcnt(uint64(uint32(n))); g != e { + t.Fatal(n, g, e) + } + + if g, e := PopCount(int(n)), bcnt(uint64(uint(n))); g != e { + t.Fatal(n, g, e) + } + + if g, e := PopCountUint(uint(n)), bcnt(uint64(uint(n))); g != e { + t.Fatal(n, g, e) + } + + if g, e := PopCountUint64(n), bcnt(n); g != e { + t.Fatal(n, g, e) + } + + if g, e := PopCountUintptr(uintptr(n)), bcnt(uint64(uintptr(n))); g != e { + t.Fatal(n, g, e) + } + } +} + +var gcds = []struct{ a, b, gcd uint64 }{ + {8, 12, 4}, + {12, 18, 6}, + {42, 56, 14}, + {54, 24, 6}, + {252, 105, 21}, + {1989, 867, 51}, + {1071, 462, 21}, + {2 * 3 * 5 * 7 * 11, 5 * 7 * 11 * 13 * 17, 5 * 7 * 11}, + {2 * 3 * 5 * 7 * 7 * 11, 5 * 7 * 7 * 11 * 13 * 17, 5 * 7 * 7 * 11}, + {2 * 3 * 5 * 7 * 7 * 11, 5 * 7 * 7 * 13 * 17, 5 * 7 * 7}, + {2 * 3 * 5 * 7 * 11, 13 * 17 * 19, 1}, +} + +func TestGCD(t *testing.T) { + for i, v := range gcds { + if v.a <= math.MaxUint16 && v.b <= math.MaxUint16 { + if g, e := uint64(GCDUint16(uint16(v.a), uint16(v.b))), v.gcd; g != e { + t.Errorf("%d: got gcd(%d, %d) %d, exp %d", i, v.a, v.b, g, e) + } + if g, e := uint64(GCDUint16(uint16(v.b), uint16(v.a))), v.gcd; g != e { + t.Errorf("%d: got gcd(%d, %d) %d, exp %d", i, v.b, v.a, g, e) + } + } + if v.a <= math.MaxUint32 && v.b <= math.MaxUint32 { + if g, e := uint64(GCDUint32(uint32(v.a), uint32(v.b))), v.gcd; g != e { + t.Errorf("%d: got gcd(%d, %d) %d, exp %d", i, v.a, v.b, g, e) + } + if g, e := uint64(GCDUint32(uint32(v.b), uint32(v.a))), v.gcd; g != e { + t.Errorf("%d: got gcd(%d, %d) %d, exp %d", i, v.b, v.a, g, e) + } + } + if g, e := GCDUint64(v.a, v.b), v.gcd; g != e { + t.Errorf("%d: got gcd(%d, %d) %d, exp %d", i, v.a, v.b, g, e) + } + if g, e := GCDUint64(v.b, v.a), v.gcd; g != e { + t.Errorf("%d: got gcd(%d, %d) %d, exp %d", i, v.b, v.a, g, e) + } + } +} + +func lg2(n uint64) (lg int) { + if n == 0 { + return -1 + } + + for n >>= 1; n != 0; n >>= 1 { + lg++ + } + return +} + +func TestLog2(t *testing.T) { + if g, e := Log2Byte(0), -1; g != e { + t.Error(g, e) + } + if g, e := Log2Uint16(0), -1; g != e { + t.Error(g, e) + } + if g, e := Log2Uint32(0), -1; g != e { + t.Error(g, e) + } + if g, e := Log2Uint64(0), -1; g != e { + t.Error(g, e) + } + const N = 1e6 + rng := r64() + for i := 0; i < N; i++ { + n := uint64(rng.Next().Int64()) + if g, e := Log2Uint64(n), lg2(n); g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := Log2Uint32(uint32(n)), lg2(n&0xffffffff); g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := Log2Uint16(uint16(n)), lg2(n&0xffff); g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := Log2Byte(byte(n)), lg2(n&0xff); g != e { + t.Fatalf("%b %d %d", n, g, e) + } + } +} + +func TestBitLen(t *testing.T) { + if g, e := BitLenByte(0), 0; g != e { + t.Error(g, e) + } + if g, e := BitLenUint16(0), 0; g != e { + t.Error(g, e) + } + if g, e := BitLenUint32(0), 0; g != e { + t.Error(g, e) + } + if g, e := BitLenUint64(0), 0; g != e { + t.Error(g, e) + } + if g, e := BitLenUintptr(0), 0; g != e { + t.Error(g, e) + } + const N = 1e6 + rng := r64() + for i := 0; i < N; i++ { + n := uint64(rng.Next().Int64()) + if g, e := BitLenUintptr(uintptr(n)), lg2(uint64(uintptr(n)))+1; g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := BitLenUint64(n), lg2(n)+1; g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := BitLenUint32(uint32(n)), lg2(n&0xffffffff)+1; g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := BitLen(int(n)), lg2(uint64(uint(n)))+1; g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := BitLenUint(uint(n)), lg2(uint64(uint(n)))+1; g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := BitLenUint16(uint16(n)), lg2(n&0xffff)+1; g != e { + t.Fatalf("%b %d %d", n, g, e) + } + if g, e := BitLenByte(byte(n)), lg2(n&0xff)+1; g != e { + t.Fatalf("%b %d %d", n, g, e) + } + } +} + +func BenchmarkGCDByte(b *testing.B) { + const N = 1 << 16 + type t byte + type u struct{ a, b t } + b.StopTimer() + rng := r32() + a := make([]u, N) + for i := range a { + a[i] = u{t(rng.Next()), t(rng.Next())} + } + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + GCDByte(byte(v.a), byte(v.b)) + } +} + +func BenchmarkGCDUint16(b *testing.B) { + const N = 1 << 16 + type t uint16 + type u struct{ a, b t } + b.StopTimer() + rng := r32() + a := make([]u, N) + for i := range a { + a[i] = u{t(rng.Next()), t(rng.Next())} + } + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + GCDUint16(uint16(v.a), uint16(v.b)) + } +} + +func BenchmarkGCDUint32(b *testing.B) { + const N = 1 << 16 + type t uint32 + type u struct{ a, b t } + b.StopTimer() + rng := r32() + a := make([]u, N) + for i := range a { + a[i] = u{t(rng.Next()), t(rng.Next())} + } + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + GCDUint32(uint32(v.a), uint32(v.b)) + } +} + +func BenchmarkGCDUint64(b *testing.B) { + const N = 1 << 16 + type t uint64 + type u struct{ a, b t } + b.StopTimer() + rng := r64() + a := make([]u, N) + for i := range a { + a[i] = u{t(rng.Next().Int64()), t(rng.Next().Int64())} + } + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + GCDUint64(uint64(v.a), uint64(v.b)) + } +} + +func BenchmarkLog2Byte(b *testing.B) { + const N = 1 << 16 + type t byte + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + Log2Byte(byte(a[i&(N-1)])) + } +} + +func BenchmarkLog2Uint16(b *testing.B) { + const N = 1 << 16 + type t uint16 + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + Log2Uint16(uint16(a[i&(N-1)])) + } +} + +func BenchmarkLog2Uint32(b *testing.B) { + const N = 1 << 16 + type t uint32 + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + Log2Uint32(uint32(a[i&(N-1)])) + } +} + +func BenchmarkLog2Uint64(b *testing.B) { + const N = 1 << 16 + type t uint64 + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + Log2Uint64(uint64(a[i&(N-1)])) + } +} +func BenchmarkBitLenByte(b *testing.B) { + const N = 1 << 16 + type t byte + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + BitLenByte(byte(a[i&(N-1)])) + } +} + +func BenchmarkBitLenUint16(b *testing.B) { + const N = 1 << 16 + type t uint16 + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + BitLenUint16(uint16(a[i&(N-1)])) + } +} + +func BenchmarkBitLenUint32(b *testing.B) { + const N = 1 << 16 + type t uint32 + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + BitLenUint32(uint32(a[i&(N-1)])) + } +} + +func BenchmarkBitLen(b *testing.B) { + const N = 1 << 16 + type t int + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + BitLen(int(a[i&(N-1)])) + } +} + +func BenchmarkBitLenUint(b *testing.B) { + const N = 1 << 16 + type t uint + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + BitLenUint(uint(a[i&(N-1)])) + } +} + +func BenchmarkBitLenUintptr(b *testing.B) { + const N = 1 << 16 + type t uintptr + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + BitLenUintptr(uintptr(a[i&(N-1)])) + } +} + +func BenchmarkBitLenUint64(b *testing.B) { + const N = 1 << 16 + type t uint64 + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + BitLenUint64(uint64(a[i&(N-1)])) + } +} + +func BenchmarkPopCountByte(b *testing.B) { + const N = 1 << 16 + type t byte + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + PopCountByte(byte(a[i&(N-1)])) + } +} + +func BenchmarkPopCountUint16(b *testing.B) { + const N = 1 << 16 + type t uint16 + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + PopCountUint16(uint16(a[i&(N-1)])) + } +} + +func BenchmarkPopCountUint32(b *testing.B) { + const N = 1 << 16 + type t uint32 + b.StopTimer() + rng := r32() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + PopCountUint32(uint32(a[i&(N-1)])) + } +} + +func BenchmarkPopCount(b *testing.B) { + const N = 1 << 16 + type t int + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + PopCount(int(a[i&(N-1)])) + } +} + +func BenchmarkPopCountUint(b *testing.B) { + const N = 1 << 16 + type t uint + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + PopCountUint(uint(a[i&(N-1)])) + } +} + +func BenchmarkPopCountUintptr(b *testing.B) { + const N = 1 << 16 + type t uintptr + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + PopCountUintptr(uintptr(a[i&(N-1)])) + } +} + +func BenchmarkPopCountUint64(b *testing.B) { + const N = 1 << 16 + type t uint64 + b.StopTimer() + rng := r64() + a := make([]t, N) + for i := range a { + a[i] = t(rng.Next().Int64()) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + PopCountUint64(uint64(a[i&(N-1)])) + } +} + +func TestUintptrBits(t *testing.T) { + switch g := UintptrBits(); g { + case 32, 64: + // ok + t.Log(g) + default: + t.Fatalf("got %d, expected 32 or 64", g) + } +} + +func BenchmarkUintptrBits(b *testing.B) { + for i := 0; i < b.N; i++ { + UintptrBits() + } +} + +func TestModPowByte(t *testing.T) { + data := []struct{ b, e, m, r byte }{ + {0, 1, 1, 0}, + {0, 2, 1, 0}, + {0, 3, 1, 0}, + + {1, 0, 1, 0}, + {1, 1, 1, 0}, + {1, 2, 1, 0}, + {1, 3, 1, 0}, + + {2, 0, 1, 0}, + {2, 1, 1, 0}, + {2, 2, 1, 0}, + {2, 3, 1, 0}, + + {2, 11, 23, 1}, // 23|M11 + {2, 11, 89, 1}, // 89|M11 + {2, 23, 47, 1}, // 47|M23 + {5, 3, 13, 8}, + } + + for _, v := range data { + if g, e := ModPowByte(v.b, v.e, v.m), v.r; g != e { + t.Errorf("b %d e %d m %d: got %d, exp %d", v.b, v.e, v.m, g, e) + } + } +} + +func TestModPowUint16(t *testing.T) { + data := []struct{ b, e, m, r uint16 }{ + {0, 1, 1, 0}, + {0, 2, 1, 0}, + {0, 3, 1, 0}, + + {1, 0, 1, 0}, + {1, 1, 1, 0}, + {1, 2, 1, 0}, + {1, 3, 1, 0}, + + {2, 0, 1, 0}, + {2, 1, 1, 0}, + {2, 2, 1, 0}, + {2, 3, 1, 0}, + + {2, 11, 23, 1}, // 23|M11 + {2, 11, 89, 1}, // 89|M11 + {2, 23, 47, 1}, // 47|M23 + {2, 929, 13007, 1}, // 13007|M929 + {4, 13, 497, 445}, + {5, 3, 13, 8}, + } + + for _, v := range data { + if g, e := ModPowUint16(v.b, v.e, v.m), v.r; g != e { + t.Errorf("b %d e %d m %d: got %d, exp %d", v.b, v.e, v.m, g, e) + } + } +} + +func TestModPowUint32(t *testing.T) { + data := []struct{ b, e, m, r uint32 }{ + {0, 1, 1, 0}, + {0, 2, 1, 0}, + {0, 3, 1, 0}, + + {1, 0, 1, 0}, + {1, 1, 1, 0}, + {1, 2, 1, 0}, + {1, 3, 1, 0}, + + {2, 0, 1, 0}, + {2, 1, 1, 0}, + {2, 2, 1, 0}, + {2, 3, 1, 0}, + + {2, 23, 47, 1}, // 47|M23 + {2, 67, 193707721, 1}, // 193707721|M67 + {2, 929, 13007, 1}, // 13007|M929 + {4, 13, 497, 445}, + {5, 3, 13, 8}, + {2, 500471, 264248689, 1}, + {2, 1000249, 112027889, 1}, + {2, 2000633, 252079759, 1}, + {2, 3000743, 222054983, 1}, + {2, 4000741, 1920355681, 1}, + {2, 5000551, 330036367, 1}, + {2, 6000479, 1020081431, 1}, + {2, 7000619, 840074281, 1}, + {2, 8000401, 624031279, 1}, + {2, 9000743, 378031207, 1}, + {2, 10000961, 380036519, 1}, + {2, 20000723, 40001447, 1}, + } + + for _, v := range data { + if g, e := ModPowUint32(v.b, v.e, v.m), v.r; g != e { + t.Errorf("b %d e %d m %d: got %d, exp %d", v.b, v.e, v.m, g, e) + } + } +} + +func TestModPowUint64(t *testing.T) { + data := []struct{ b, e, m, r uint64 }{ + {0, 1, 1, 0}, + {0, 2, 1, 0}, + {0, 3, 1, 0}, + + {1, 0, 1, 0}, + {1, 1, 1, 0}, + {1, 2, 1, 0}, + {1, 3, 1, 0}, + + {2, 0, 1, 0}, + {2, 1, 1, 0}, + {2, 2, 1, 0}, + {2, 3, 1, 0}, + + {2, 23, 47, 1}, // 47|M23 + {2, 67, 193707721, 1}, // 193707721|M67 + {2, 929, 13007, 1}, // 13007|M929 + {4, 13, 497, 445}, + {5, 3, 13, 8}, + {2, 500471, 264248689, 1}, // m|Me ... + {2, 1000249, 112027889, 1}, + {2, 2000633, 252079759, 1}, + {2, 3000743, 222054983, 1}, + {2, 4000741, 1920355681, 1}, + {2, 5000551, 330036367, 1}, + {2, 6000479, 1020081431, 1}, + {2, 7000619, 840074281, 1}, + {2, 8000401, 624031279, 1}, + {2, 9000743, 378031207, 1}, + {2, 10000961, 380036519, 1}, + {2, 20000723, 40001447, 1}, + {2, 1000099, 1872347344039, 1}, + + {9223372036854775919, 9223372036854776030, 9223372036854776141, 7865333882915297658}, + } + + for _, v := range data { + if g, e := ModPowUint64(v.b, v.e, v.m), v.r; g != e { + t.Errorf("b %d e %d m %d: got %d, exp %d", v.b, v.e, v.m, g, e) + } + } +} + +func TestModPowBigInt(t *testing.T) { + data := []struct { + b, e int64 + m interface{} + r int64 + }{ + {0, 1, 1, 0}, + {0, 2, 1, 0}, + {0, 3, 1, 0}, + + {1, 0, 1, 0}, + {1, 1, 1, 0}, + {1, 2, 1, 0}, + {1, 3, 1, 0}, + + {2, 0, 1, 0}, + {2, 1, 1, 0}, + {2, 2, 1, 0}, + {2, 3, 1, 0}, + + {2, 23, 47, 1}, // 47|M23 + {2, 67, 193707721, 1}, // 193707721|M67 + {2, 929, 13007, 1}, // 13007|M929 + {4, 13, 497, 445}, + {5, 3, 13, 8}, + {2, 500471, 264248689, 1}, // m|Me ... + {2, 1000249, 112027889, 1}, + {2, 2000633, 252079759, 1}, + {2, 3000743, 222054983, 1}, + {2, 4000741, 1920355681, 1}, + {2, 5000551, 330036367, 1}, + {2, 6000479, 1020081431, 1}, + {2, 7000619, 840074281, 1}, + {2, 8000401, 624031279, 1}, + {2, 9000743, 378031207, 1}, + {2, 10000961, 380036519, 1}, + {2, 20000723, 40001447, 1}, + {2, 100279, "11502865265922183403581252152383", 1}, + + {2, 7293457, "533975545077050000610542659519277030089249998649", 1}, + } + + for _, v := range data { + var m big.Int + switch x := v.m.(type) { + case int: + m.SetInt64(int64(x)) + case string: + m.SetString(x, 10) + } + b, e, r := big.NewInt(v.b), big.NewInt(v.e), big.NewInt(v.r) + if g, e := ModPowBigInt(b, e, &m), r; g.Cmp(e) != 0 { + t.Errorf("b %s e %s m %v: got %s, exp %s", b, e, m, g, e) + } + } + + s := func(n string) *big.Int { + i, ok := big.NewInt(0).SetString(n, 10) + if !ok { + t.Fatal(ok) + } + + return i + } + + if g, e := ModPowBigInt( + s("36893488147419103343"), s("36893488147419103454"), s("36893488147419103565")), s("34853007610367449339"); g.Cmp(e) != 0 { + t.Fatal(g, e) + } +} + +func BenchmarkModPowByte(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ b, e, m byte } + a := make([]t, N) + r := r32() + for i := range a { + a[i] = t{ + byte(r.Next() | 2), + byte(r.Next() | 2), + byte(r.Next() | 2), + } + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + ModPowByte(v.b, v.e, v.m) + } +} + +func BenchmarkModPowUint16(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ b, e, m uint16 } + a := make([]t, N) + r := r32() + for i := range a { + a[i] = t{ + uint16(r.Next() | 2), + uint16(r.Next() | 2), + uint16(r.Next() | 2), + } + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + ModPowUint16(v.b, v.e, v.m) + } +} + +func BenchmarkModPowUint32(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ b, e, m uint32 } + a := make([]t, N) + r := r32() + for i := range a { + a[i] = t{ + uint32(r.Next() | 2), + uint32(r.Next() | 2), + uint32(r.Next() | 2), + } + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + ModPowUint32(v.b, v.e, v.m) + } +} + +func BenchmarkModPowUint64(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ b, e, m uint64 } + a := make([]t, N) + r := r64() + for i := range a { + a[i] = t{ + uint64(r.Next().Int64() | 2), + uint64(r.Next().Int64() | 2), + uint64(r.Next().Int64() | 2), + } + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + ModPowUint64(v.b, v.e, v.m) + } +} + +func BenchmarkModPowBigInt(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ b, e, m *big.Int } + a := make([]t, N) + mx := big.NewInt(math.MaxInt64) + mx.Mul(mx, mx) + r, err := NewFCBig(big.NewInt(2), mx, true) + if err != nil { + b.Fatal(err) + } + for i := range a { + a[i] = t{ + r.Next(), + r.Next(), + r.Next(), + } + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + ModPowBigInt(v.b, v.e, v.m) + } +} + +func TestAdd128(t *testing.T) { + const N = 1e5 + r := r64() + var mm big.Int + for i := 0; i < N; i++ { + a, b := uint64(r.Next().Int64()), uint64(r.Next().Int64()) + aa, bb := big.NewInt(0).SetUint64(a), big.NewInt(0).SetUint64(b) + mhi, mlo := AddUint128_64(a, b) + m := big.NewInt(0).SetUint64(mhi) + m.Lsh(m, 64) + m.Add(m, big.NewInt(0).SetUint64(mlo)) + mm.Add(aa, bb) + if m.Cmp(&mm) != 0 { + t.Fatalf("%d\na %40d\nb %40d\ng %40s %032x\ne %40s %032x", i, a, b, m, m, &mm, &mm) + } + } +} + +func TestMul128(t *testing.T) { + const N = 1e5 + r := r64() + var mm big.Int + f := func(a, b uint64) { + aa, bb := big.NewInt(0).SetUint64(a), big.NewInt(0).SetUint64(b) + mhi, mlo := MulUint128_64(a, b) + m := big.NewInt(0).SetUint64(mhi) + m.Lsh(m, 64) + m.Add(m, big.NewInt(0).SetUint64(mlo)) + mm.Mul(aa, bb) + if m.Cmp(&mm) != 0 { + t.Fatalf("\na %40d\nb %40d\ng %40s %032x\ne %40s %032x", a, b, m, m, &mm, &mm) + } + } + for i := 0; i < N; i++ { + f(uint64(r.Next().Int64()), uint64(r.Next().Int64())) + } + for x := 0; x <= 1<<9; x++ { + for y := 0; y <= 1<<9; y++ { + f(math.MaxUint64-uint64(x), math.MaxUint64-uint64(y)) + } + } +} + +func BenchmarkMul128(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ a, b uint64 } + a := make([]t, N) + r := r64() + for i := range a { + a[i] = t{ + uint64(r.Next().Int64()), + uint64(r.Next().Int64()), + } + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + MulUint128_64(v.a, v.b) + } +} + +func BenchmarkMul128Big(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ a, b *big.Int } + a := make([]t, N) + r := r64() + for i := range a { + a[i] = t{ + big.NewInt(r.Next().Int64()), + big.NewInt(r.Next().Int64()), + } + } + var x big.Int + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + x.Mul(v.a, v.b) + } +} + +func TestIsPrimeUint64(t *testing.T) { + f := func(lo, hi uint64, exp int) { + got := 0 + for n := lo; n <= hi; { + if IsPrimeUint64(n) { + got++ + } + n0 := n + n++ + if n < n0 { + break + } + } + if got != exp { + t.Fatal(lo, hi, got, exp) + } + } + + // lo, hi, PrimePi(hi)-PrimePi(lo) + f(0, 1e4, 1229) + f(1e5, 1e5+1e4, 861) + f(1e6, 1e6+1e4, 753) + f(1e7, 1e7+1e4, 614) + f(1e8, 1e8+1e4, 551) + f(1e9, 1e9+1e4, 487) + f(1e10, 1e10+1e4, 406) + f(1e11, 1e11+1e4, 394) + f(1e12, 1e12+1e4, 335) + f(1e13, 1e13+1e4, 354) + f(1e14, 1e14+1e4, 304) + f(1e15, 1e15+1e4, 263) + f(1e16, 1e16+1e4, 270) + f(1e17, 1e17+1e4, 265) + f(1e18, 1e18+1e4, 241) + f(1e19, 1e19+1e4, 255) + f(1<<64-1e4, 1<<64-1, 218) +} + +func TestProbablyPrimeUint32(t *testing.T) { + f := func(n, firstFail uint32, primes []uint32) { + for ; n <= firstFail; n += 2 { + prp := true + for _, a := range primes { + if !ProbablyPrimeUint32(n, a) { + prp = false + break + } + } + if prp != IsPrime(n) && n != firstFail { + t.Fatal(n) + } + } + } + if !ProbablyPrimeUint32(5, 2) { + t.Fatal(false) + } + if !ProbablyPrimeUint32(7, 2) { + t.Fatal(false) + } + if ProbablyPrimeUint32(9, 2) { + t.Fatal(true) + } + if !ProbablyPrimeUint32(11, 2) { + t.Fatal(false) + } + // http://oeis.org/A014233 + f(5, 2047, []uint32{2}) + f(2047, 1373653, []uint32{2, 3}) + f(1373653, 25326001, []uint32{2, 3, 5}) +} + +func BenchmarkProbablyPrimeUint32(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ n, a uint32 } + data := make([]t, N) + r := r32() + for i := range data { + n := uint32(r.Next()) | 1 + if n <= 3 { + n += 5 + } + a := uint32(r.Next()) + if a <= 1 { + a += 2 + } + data[i] = t{n, a} + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := data[i&(N-1)] + ProbablyPrimeUint32(v.n, v.a) + } +} + +func TestProbablyPrimeUint64_32(t *testing.T) { + f := func(n, firstFail uint64, primes []uint32) { + for ; n <= firstFail; n += 2 { + prp := true + for _, a := range primes { + if !ProbablyPrimeUint64_32(n, a) { + prp = false + break + } + } + if prp != IsPrimeUint64(n) && n != firstFail { + t.Fatal(n) + } + } + } + if !ProbablyPrimeUint64_32(5, 2) { + t.Fatal(false) + } + if !ProbablyPrimeUint64_32(7, 2) { + t.Fatal(false) + } + if ProbablyPrimeUint64_32(9, 2) { + t.Fatal(true) + } + if !ProbablyPrimeUint64_32(11, 2) { + t.Fatal(false) + } + // http://oeis.org/A014233 + f(5, 2047, []uint32{2}) + f(2047, 1373653, []uint32{2, 3}) +} + +func BenchmarkProbablyPrimeUint64_32(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct { + n uint64 + a uint32 + } + data := make([]t, N) + r := r32() + r2 := r64() + for i := range data { + var n uint64 + for n <= 3 { + n = uint64(r2.Next().Int64()) | 1 + } + var a uint32 + for a <= 1 { + a = uint32(r.Next()) + } + data[i] = t{n, a} + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := data[i&(N-1)] + ProbablyPrimeUint64_32(v.n, v.a) + } +} + +func TestProbablyPrimeBigInt_32(t *testing.T) { + f := func(n0, firstFail0 uint64, primes []uint32) { + n, firstFail := big.NewInt(0).SetUint64(n0), big.NewInt(0).SetUint64(firstFail0) + for ; n.Cmp(firstFail) <= 0; n.Add(n, _2) { + prp := true + for _, a := range primes { + if !ProbablyPrimeBigInt_32(n, a) { + prp = false + break + } + } + if prp != IsPrimeUint64(n0) && n0 != firstFail0 { + t.Fatal(n) + } + n0 += 2 + } + } + if !ProbablyPrimeBigInt_32(big.NewInt(5), 2) { + t.Fatal(false) + } + if !ProbablyPrimeBigInt_32(big.NewInt(7), 2) { + t.Fatal(false) + } + if ProbablyPrimeBigInt_32(big.NewInt(9), 2) { + t.Fatal(true) + } + if !ProbablyPrimeBigInt_32(big.NewInt(11), 2) { + t.Fatal(false) + } + // http://oeis.org/A014233 + f(5, 2047, []uint32{2}) + f(2047, 1373653, []uint32{2, 3}) +} + +func BenchmarkProbablyPrimeBigInt_32(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct { + n *big.Int + a uint32 + } + data := make([]t, N) + r := r32() + r2 := r64() + for i := range data { + var n uint64 + for n <= 3 { + n = uint64(r2.Next().Int64()) | 1 + } + var a uint32 + for a <= 1 { + a = uint32(r.Next()) + } + data[i] = t{big.NewInt(0).SetUint64(n), a} + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := data[i&(N-1)] + ProbablyPrimeBigInt_32(v.n, v.a) + } +} + +func TestProbablyPrimeBigInt(t *testing.T) { + f := func(n0, firstFail0 uint64, primes []uint32) { + n, firstFail := big.NewInt(0).SetUint64(n0), big.NewInt(0).SetUint64(firstFail0) + for ; n.Cmp(firstFail) <= 0; n.Add(n, _2) { + prp := true + var a big.Int + for _, a0 := range primes { + a.SetInt64(int64(a0)) + if !ProbablyPrimeBigInt(n, &a) { + prp = false + break + } + } + if prp != IsPrimeUint64(n0) && n0 != firstFail0 { + t.Fatal(n) + } + n0 += 2 + } + } + if !ProbablyPrimeBigInt(big.NewInt(5), _2) { + t.Fatal(false) + } + if !ProbablyPrimeBigInt(big.NewInt(7), _2) { + t.Fatal(false) + } + if ProbablyPrimeBigInt(big.NewInt(9), _2) { + t.Fatal(true) + } + if !ProbablyPrimeBigInt(big.NewInt(11), _2) { + t.Fatal(false) + } + // http://oeis.org/A014233 + f(5, 2047, []uint32{2}) + f(2047, 1373653, []uint32{2, 3}) +} + +var once2059 sync.Once + +func BenchmarkProbablyPrimeBigInt64(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + once2059.Do(func() { b.Log("64 bit n, 64 bit a\n") }) + type t struct { + n, a *big.Int + } + data := make([]t, N) + r := r64() + for i := range data { + var n uint64 + for n <= 3 { + n = uint64(r.Next().Int64()) | 1 + } + var a uint64 + for a <= 1 { + a = uint64(r.Next().Int64()) + } + data[i] = t{big.NewInt(0).SetUint64(n), big.NewInt(0).SetUint64(uint64(a))} + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := data[i&(N-1)] + ProbablyPrimeBigInt(v.n, v.a) + } +} + +var once2090 sync.Once + +func BenchmarkProbablyPrimeBigInt128(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + once2090.Do(func() { b.Log("128 bit n, 128 bit a\n") }) + type t struct { + n, a *big.Int + } + data := make([]t, N) + r := r64() + for i := range data { + n := big.NewInt(0).SetUint64(uint64(r.Next().Int64())) + n.Lsh(n, 64) + n.Add(n, big.NewInt(0).SetUint64(uint64(r.Next().Int64())|1)) + a := big.NewInt(0).SetUint64(uint64(r.Next().Int64())) + a.Lsh(a, 64) + a.Add(a, big.NewInt(0).SetUint64(uint64(r.Next().Int64()))) + data[i] = t{n, a} + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := data[i&(N-1)] + ProbablyPrimeBigInt(v.n, v.a) + } +} + +func TestQCmpUint32(t *testing.T) { + const N = 6e4 + r := r32() + var x, y big.Rat + for i := 0; i < N; i++ { + a, b, c, d := uint32(r.Next()), uint32(r.Next()), uint32(r.Next()), uint32(r.Next()) + x.SetFrac64(int64(a), int64(b)) + y.SetFrac64(int64(c), int64(d)) + if g, e := QCmpUint32(a, b, c, d), x.Cmp(&y); g != e { + t.Fatal(a, b, c, d, g, e) + } + } +} + +func TestQScaleUint32(t *testing.T) { + const N = 4e4 + r := r32() + var x, y big.Rat + var a uint64 + var b, c, d uint32 + for i := 0; i < N; i++ { + for { + b, c, d = uint32(r.Next()), uint32(r.Next()), uint32(r.Next()) + a = QScaleUint32(b, c, d) + if a <= math.MaxInt64 { + break + } + } + x.SetFrac64(int64(a), int64(b)) + y.SetFrac64(int64(c), int64(d)) + if g := x.Cmp(&y); g < 0 { + t.Fatal(a, b, c, d, g, "expexted 1 or 0") + } + + if a != 0 { + x.SetFrac64(int64(a-1), int64(b)) + if g := x.Cmp(&y); g > 0 { + t.Fatal(a, b, c, d, g, "expected -1 or 0") + } + } + } +} + +var smalls = []uint32{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + +func isPrimorialProduct(t FactorTerms, maxp uint32) bool { + if len(t) == 0 { + return false + } + + pmax := uint32(32) + for i, v := range t { + if v.Prime != smalls[i] || v.Power > pmax || v.Power > maxp { + return false + } + pmax = v.Power + } + return true +} + +func TestPrimorialProductsUint32(t *testing.T) { + r := PrimorialProductsUint32(2*3*5*7*11*13*17*19+1, math.MaxUint32, 1) + if len(r) != 1 { + t.Fatal(len(r)) + } + + if r[0] != 2*3*5*7*11*13*17*19*23 { + t.Fatal(r[0]) + } + + r = PrimorialProductsUint32(0, math.MaxUint32, math.MaxUint32) + if g, e := len(r), 1679; g != e { + t.Fatal(g, e) + } + + m := map[uint32]struct{}{} + for _, v := range r { + if _, ok := m[v]; ok { + t.Fatal(v) + } + + m[v] = struct{}{} + } + + for lo := uint32(0); lo < 5e4; lo += 1e3 { + hi := 1e2 * lo + for max := uint32(0); max <= 32; max++ { + m := map[uint32]struct{}{} + for i, v := range PrimorialProductsUint32(lo, hi, max) { + f := FactorInt(v) + if v < lo || v > hi { + t.Fatal(lo, hi, max, v) + } + + if _, ok := m[v]; ok { + t.Fatal(i, lo, hi, max, v, f) + } + + m[v] = struct{}{} + if !isPrimorialProduct(f, max) { + t.Fatal(i, v) + } + + for _, v := range f { + if v.Power > max { + t.Fatal(i, v, f) + } + } + } + } + } +} + +func BenchmarkPrimorialProductsUint32(b *testing.B) { + for i := 0; i < b.N; i++ { + PrimorialProductsUint32(0, math.MaxUint32, math.MaxUint32) + } +} + +func powerizeUint32BigInt(b uint32, n *big.Int) (e uint32, p *big.Int) { + p = big.NewInt(1) + bb := big.NewInt(int64(b)) + for p.Cmp(n) < 0 { + p.Mul(p, bb) + e++ + } + return +} + +func TestPowerizeUint32BigInt(t *testing.T) { + var data = []struct{ b, n, e, p int }{ + {0, 10, 0, -1}, + {1, 10, 0, -1}, + {2, -1, 0, -1}, + {2, 0, 0, 1}, + {2, 1, 0, 1}, + {2, 2, 1, 2}, + {2, 3, 2, 4}, + {3, 0, 0, 1}, + {3, 1, 0, 1}, + {3, 2, 1, 3}, + {3, 3, 1, 3}, + {3, 4, 2, 9}, + {3, 8, 2, 9}, + {3, 9, 2, 9}, + {3, 10, 3, 27}, + {3, 80, 4, 81}, + } + + var n big.Int + for _, v := range data { + b := v.b + n.SetInt64(int64(v.n)) + e, p := PowerizeUint32BigInt(uint32(b), &n) + if e != uint32(v.e) { + t.Fatal(b, &n, e, p, v.e, v.p) + } + + if v.p < 0 { + if p != nil { + t.Fatal(b, &n, e, p, v.e, v.p) + } + continue + } + + if p.Int64() != int64(v.p) { + t.Fatal(b, &n, e, p, v.e, v.p) + } + } + const N = 1e5 + var nn big.Int + for _, base := range []uint32{2, 3, 15, 17} { + for n := 0; n <= N; n++ { + nn.SetInt64(int64(n)) + ge, gp := PowerizeUint32BigInt(base, &nn) + ee, ep := powerizeUint32BigInt(base, &nn) + if ge != ee || gp.Cmp(ep) != 0 { + t.Fatal(base, n, ge, gp, ee, ep) + } + + gp.Div(gp, big.NewInt(int64(base))) + if gp.Sign() > 0 && gp.Cmp(&nn) >= 0 { + t.Fatal(gp.Sign(), gp.Cmp(&nn)) + } + } + } +} + +func benchmarkPowerizeUint32BigInt(b *testing.B, base uint32, exp int) { + b.StopTimer() + var n big.Int + n.SetBit(&n, exp, 1) + b.StartTimer() + for i := 0; i < b.N; i++ { + PowerizeUint32BigInt(base, &n) + } +} + +func BenchmarkPowerizeUint32BigInt_2_2e1e1(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 2, 1e1) +} + +func BenchmarkPowerizeUint32BigInt_2_2e1e2(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 2, 1e2) +} + +func BenchmarkPowerizeUint32BigInt_2_2e1e3(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 2, 1e3) +} + +func BenchmarkPowerizeUint32BigInt_2_2e1e4(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 2, 1e4) +} + +func BenchmarkPowerizeUint32BigInt_2_2e1e5(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 2, 1e5) +} + +func BenchmarkPowerizeUint32BigInt_2_2e1e6(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 2, 1e6) +} + +func BenchmarkPowerizeUint32BigInt_2_2e1e7(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 2, 1e7) +} + +func BenchmarkPowerizeUint32BigInt_3_2e1e1(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 3, 1e1) +} + +func BenchmarkPowerizeUint32BigInt_3_2e1e2(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 3, 1e2) +} + +func BenchmarkPowerizeUint32BigInt_3_2e1e3(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 3, 1e3) +} + +func BenchmarkPowerizeUint32BigInt_3_2e1e4(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 3, 1e4) +} + +func BenchmarkPowerizeUint32BigInt_3_2e1e5(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 3, 1e5) +} + +func BenchmarkPowerizeUint32BigInt_3_2e1e6(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 3, 1e6) +} + +func BenchmarkPowerizeUint32BigInt_15_2e1e1(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 15, 1e1) +} + +func BenchmarkPowerizeUint32BigInt_15_2e1e2(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 15, 1e2) +} + +func BenchmarkPowerizeUint32BigInt_15_2e1e3(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 15, 1e3) +} + +func BenchmarkPowerizeUint32BigInt_15_2e1e4(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 15, 1e4) +} + +func BenchmarkPowerizeUint32BigInt_15_2e1e5(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 15, 1e5) +} + +func BenchmarkPowerizeUint32BigInt_15_2e1e6(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 15, 1e6) +} + +func BenchmarkPowerizeUint32BigInt_17_2e1e1(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 17, 1e1) +} + +func BenchmarkPowerizeUint32BigInt_17_2e1e2(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 17, 1e2) +} + +func BenchmarkPowerizeUint32BigInt_17_2e1e3(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 17, 1e3) +} + +func BenchmarkPowerizeUint32BigInt_17_2e1e4(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 17, 1e4) +} + +func BenchmarkPowerizeUint32BigInt_17_2e1e5(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 17, 1e5) +} + +func BenchmarkPowerizeUint32BigInt_17_2e1e6(b *testing.B) { + benchmarkPowerizeUint32BigInt(b, 17, 1e6) +} + +func TestPowerizeBigInt(t *testing.T) { + var data = []struct{ b, n, e, p int }{ + {0, 10, 0, -1}, + {1, 10, 0, -1}, + {2, -1, 0, -1}, + {2, 0, 0, 1}, + {2, 1, 0, 1}, + {2, 2, 1, 2}, + {2, 3, 2, 4}, + {3, 0, 0, 1}, + {3, 1, 0, 1}, + {3, 2, 1, 3}, + {3, 3, 1, 3}, + {3, 4, 2, 9}, + {3, 8, 2, 9}, + {3, 9, 2, 9}, + {3, 10, 3, 27}, + {3, 80, 4, 81}, + } + + var b, n big.Int + for _, v := range data { + b.SetInt64(int64(v.b)) + n.SetInt64(int64(v.n)) + e, p := PowerizeBigInt(&b, &n) + if e != uint32(v.e) { + t.Fatal(&b, &n, e, p, v.e, v.p) + } + + if v.p < 0 { + if p != nil { + t.Fatal(&b, &n, e, p, v.e, v.p) + } + continue + } + + if p.Int64() != int64(v.p) { + t.Fatal(&b, &n, e, p, v.e, v.p) + } + } + const N = 1e5 + var nn big.Int + for _, base := range []uint32{2, 3, 15, 17} { + b.SetInt64(int64(base)) + for n := 0; n <= N; n++ { + nn.SetInt64(int64(n)) + ge, gp := PowerizeBigInt(&b, &nn) + ee, ep := powerizeUint32BigInt(base, &nn) + if ge != ee || gp.Cmp(ep) != 0 { + t.Fatal(base, n, ge, gp, ee, ep) + } + + gp.Div(gp, &b) + if gp.Sign() > 0 && gp.Cmp(&nn) >= 0 { + t.Fatal(gp.Sign(), gp.Cmp(&nn)) + } + } + } +} + +func benchmarkPowerizeBigInt(b *testing.B, base uint32, exp int) { + b.StopTimer() + var bb, n big.Int + n.SetBit(&n, exp, 1) + bb.SetInt64(int64(base)) + b.StartTimer() + for i := 0; i < b.N; i++ { + PowerizeBigInt(&bb, &n) + } +} + +func BenchmarkPowerizeBigInt_2_2e1e1(b *testing.B) { + benchmarkPowerizeBigInt(b, 2, 1e1) +} + +func BenchmarkPowerizeBigInt_2_2e1e2(b *testing.B) { + benchmarkPowerizeBigInt(b, 2, 1e2) +} + +func BenchmarkPowerizeBigInt_2_2e1e3(b *testing.B) { + benchmarkPowerizeBigInt(b, 2, 1e3) +} + +func BenchmarkPowerizeBigInt_2_2e1e4(b *testing.B) { + benchmarkPowerizeBigInt(b, 2, 1e4) +} + +func BenchmarkPowerizeBigInt_2_2e1e5(b *testing.B) { + benchmarkPowerizeBigInt(b, 2, 1e5) +} + +func BenchmarkPowerizeBigInt_2_2e1e6(b *testing.B) { + benchmarkPowerizeBigInt(b, 2, 1e6) +} + +func BenchmarkPowerizeBigInt_2_2e1e7(b *testing.B) { + benchmarkPowerizeBigInt(b, 2, 1e7) +} + +func BenchmarkPowerizeBigInt_3_2e1e1(b *testing.B) { + benchmarkPowerizeBigInt(b, 3, 1e1) +} + +func BenchmarkPowerizeBigInt_3_2e1e2(b *testing.B) { + benchmarkPowerizeBigInt(b, 3, 1e2) +} + +func BenchmarkPowerizeBigInt_3_2e1e3(b *testing.B) { + benchmarkPowerizeBigInt(b, 3, 1e3) +} + +func BenchmarkPowerizeBigInt_3_2e1e4(b *testing.B) { + benchmarkPowerizeBigInt(b, 3, 1e4) +} + +func BenchmarkPowerizeBigInt_3_2e1e5(b *testing.B) { + benchmarkPowerizeBigInt(b, 3, 1e5) +} + +func BenchmarkPowerizeBigInt_3_2e1e6(b *testing.B) { + benchmarkPowerizeBigInt(b, 3, 1e6) +} + +func BenchmarkPowerizeBigInt_15_2e1e1(b *testing.B) { + benchmarkPowerizeBigInt(b, 15, 1e1) +} + +func BenchmarkPowerizeBigInt_15_2e1e2(b *testing.B) { + benchmarkPowerizeBigInt(b, 15, 1e2) +} + +func BenchmarkPowerizeBigInt_15_2e1e3(b *testing.B) { + benchmarkPowerizeBigInt(b, 15, 1e3) +} + +func BenchmarkPowerizeBigInt_15_2e1e4(b *testing.B) { + benchmarkPowerizeBigInt(b, 15, 1e4) +} + +func BenchmarkPowerizeBigInt_15_2e1e5(b *testing.B) { + benchmarkPowerizeBigInt(b, 15, 1e5) +} + +func BenchmarkPowerizeBigInt_15_2e1e6(b *testing.B) { + benchmarkPowerizeBigInt(b, 15, 1e6) +} + +func BenchmarkPowerizeBigInt_17_2e1e1(b *testing.B) { + benchmarkPowerizeBigInt(b, 17, 1e1) +} + +func BenchmarkPowerizeBigInt_17_2e1e2(b *testing.B) { + benchmarkPowerizeBigInt(b, 17, 1e2) +} + +func BenchmarkPowerizeBigInt_17_2e1e3(b *testing.B) { + benchmarkPowerizeBigInt(b, 17, 1e3) +} + +func BenchmarkPowerizeBigInt_17_2e1e4(b *testing.B) { + benchmarkPowerizeBigInt(b, 17, 1e4) +} + +func BenchmarkPowerizeBigInt_17_2e1e5(b *testing.B) { + benchmarkPowerizeBigInt(b, 17, 1e5) +} + +func BenchmarkPowerizeBigInt_17_2e1e6(b *testing.B) { + benchmarkPowerizeBigInt(b, 17, 1e6) +} + +func TestEnvelope(t *testing.T) { + const prec = 1e-3 + type check struct { + approx Approximation + x, y float64 + } + data := []struct { + points []float64 + checks []check + }{ + { + []float64{0, 1}, + []check{ + {Linear, 0, 0}, + {Linear, 0.25, 0.25}, + {Linear, 0.5, 0.5}, + {Linear, 0.75, 0.75}, + {Linear, 0.9999, 1}, + }, + }, + { + []float64{-1, 0}, + []check{ + {Linear, 0, -1}, + {Linear, 0.25, -0.75}, + {Linear, 0.5, -0.5}, + {Linear, 0.75, -0.25}, + {Linear, 0.9999, 0}, + }, + }, + { + []float64{-1, 1}, + []check{ + {Linear, 0, -1}, + {Linear, 0.25, -0.5}, + {Linear, 0.5, 0}, + {Linear, 0.75, 0.5}, + {Linear, 0.9999, 1}, + }, + }, + { + []float64{-1, 1, -2}, + []check{ + {Linear, 0, -1}, + {Linear, 0.25, 0}, + {Linear, 0.5, 1}, + {Linear, 0.75, -0.5}, + {Linear, 0.9, -1.4}, + {Linear, 0.9999, -2}, + }, + }, + { + []float64{-1, 1}, + []check{ + {Sinusoidal, 0, -1}, + {Sinusoidal, 0.25, -math.Sqrt2 / 2}, + {Sinusoidal, 0.5, 0}, + {Sinusoidal, 0.75, math.Sqrt2 / 2}, + {Sinusoidal, 0.9999, 1}, + }, + }, + { + []float64{-1, 1, -2}, + []check{ + {Sinusoidal, 0, -1}, + {Sinusoidal, 1. / 8, -math.Sqrt2 / 2}, + {Sinusoidal, 2. / 8, 0}, + {Sinusoidal, 3. / 8, math.Sqrt2 / 2}, + {Sinusoidal, 4. / 8, 1}, + {Sinusoidal, 5. / 8, (3*math.Sqrt2 - 2) / 4}, + {Sinusoidal, 6. / 8, -0.5}, + {Sinusoidal, 7. / 8, (-3*math.Sqrt2 - 2) / 4}, + {Sinusoidal, 0.9999, -2}, + }, + }, + } + for i, suite := range data { + for j, test := range suite.checks { + e, g := test.y, Envelope(test.x, suite.points, test.approx) + d := math.Abs(e - g) + if d > prec { + t.Errorf( + "i %d, j %d, x %v, e %v, g %v, d %v, prec %v", + i, j, test.x, e, g, d, prec, + ) + } + } + } +} + +func TestMaxInt(t *testing.T) { + n := int64(MaxInt) + if n != math.MaxInt32 && n != math.MaxInt64 { + t.Error(n) + } + + t.Logf("64 bit ints: %t, MaxInt: %d", n == math.MaxInt64, n) +} + +func TestMinInt(t *testing.T) { + n := int64(MinInt) + if n != math.MinInt32 && n != math.MinInt64 { + t.Error(n) + } + + t.Logf("64 bit ints: %t. MinInt: %d", n == math.MinInt64, n) +} + +func TestMaxUint(t *testing.T) { + n := uint64(MaxUint) + if n != math.MaxUint32 && n != math.MaxUint64 { + t.Error(n) + } + + t.Logf("64 bit uints: %t. MaxUint: %d", n == math.MaxUint64, n) +} + +func TestMax(t *testing.T) { + tests := []struct{ a, b, e int }{ + {MinInt, MinIntM1, MaxInt}, + {MinIntM1, MinInt, MaxInt}, + {MinIntM1, MinIntM1, MaxInt}, + + {MinInt, MinInt, MinInt}, + {MinInt + 1, MinInt, MinInt + 1}, + {MinInt, MinInt + 1, MinInt + 1}, + + {-1, -1, -1}, + {-1, 0, 0}, + {-1, 1, 1}, + + {0, -1, 0}, + {0, 0, 0}, + {0, 1, 1}, + + {1, -1, 1}, + {1, 0, 1}, + {1, 1, 1}, + + {MaxInt, MaxInt, MaxInt}, + {MaxInt - 1, MaxInt, MaxInt}, + {MaxInt, MaxInt - 1, MaxInt}, + + {MaxIntP1, MaxInt, MaxInt}, + {MaxInt, MaxIntP1, MaxInt}, + {MaxIntP1, MaxIntP1, MinInt}, + } + + for _, test := range tests { + if g, e := Max(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMin(t *testing.T) { + tests := []struct{ a, b, e int }{ + {MinIntM1, MinInt, MinInt}, + {MinInt, MinIntM1, MinInt}, + {MinIntM1, MinIntM1, MaxInt}, + + {MinInt, MinInt, MinInt}, + {MinInt + 1, MinInt, MinInt}, + {MinInt, MinInt + 1, MinInt}, + + {-1, -1, -1}, + {-1, 0, -1}, + {-1, 1, -1}, + + {0, -1, -1}, + {0, 0, 0}, + {0, 1, 0}, + + {1, -1, -1}, + {1, 0, 0}, + {1, 1, 1}, + + {MaxInt, MaxInt, MaxInt}, + {MaxInt - 1, MaxInt, MaxInt - 1}, + {MaxInt, MaxInt - 1, MaxInt - 1}, + + {MaxIntP1, MaxInt, MinInt}, + {MaxInt, MaxIntP1, MinInt}, + {MaxIntP1, MaxIntP1, MinInt}, + } + + for _, test := range tests { + if g, e := Min(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestUMax(t *testing.T) { + tests := []struct{ a, b, e uint }{ + {0, 0, 0}, + {0, 1, 1}, + {1, 0, 1}, + + {10, 10, 10}, + {10, 11, 11}, + {11, 10, 11}, + {11, 11, 11}, + + {MaxUint, MaxUint, MaxUint}, + {MaxUint, MaxUint - 1, MaxUint}, + {MaxUint - 1, MaxUint, MaxUint}, + {MaxUint - 1, MaxUint - 1, MaxUint - 1}, + + {MaxUint, MaxUintP1, MaxUint}, + {MaxUintP1, MaxUint, MaxUint}, + {MaxUintP1, MaxUintP1, 0}, + } + + for _, test := range tests { + if g, e := UMax(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestUMin(t *testing.T) { + tests := []struct{ a, b, e uint }{ + {0, 0, 0}, + {0, 1, 0}, + {1, 0, 0}, + + {10, 10, 10}, + {10, 11, 10}, + {11, 10, 10}, + {11, 11, 11}, + + {MaxUint, MaxUint, MaxUint}, + {MaxUint, MaxUint - 1, MaxUint - 1}, + {MaxUint - 1, MaxUint, MaxUint - 1}, + {MaxUint - 1, MaxUint - 1, MaxUint - 1}, + + {MaxUint, MaxUintP1, 0}, + {MaxUintP1, MaxUint, 0}, + {MaxUintP1, MaxUintP1, 0}, + } + + for _, test := range tests { + if g, e := UMin(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxByte(t *testing.T) { + tests := []struct{ a, b, e byte }{ + {0, 0, 0}, + {0, 1, 1}, + {1, 0, 1}, + + {10, 10, 10}, + {10, 11, 11}, + {11, 10, 11}, + {11, 11, 11}, + + {math.MaxUint8, math.MaxUint8, math.MaxUint8}, + {math.MaxUint8, math.MaxUint8 - 1, math.MaxUint8}, + {math.MaxUint8 - 1, math.MaxUint8, math.MaxUint8}, + {math.MaxUint8 - 1, math.MaxUint8 - 1, math.MaxUint8 - 1}, + } + + for _, test := range tests { + if g, e := MaxByte(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinByte(t *testing.T) { + tests := []struct{ a, b, e byte }{ + {0, 0, 0}, + {0, 1, 0}, + {1, 0, 0}, + + {10, 10, 10}, + {10, 11, 10}, + {11, 10, 10}, + {11, 11, 11}, + + {math.MaxUint8, math.MaxUint8, math.MaxUint8}, + {math.MaxUint8, math.MaxUint8 - 1, math.MaxUint8 - 1}, + {math.MaxUint8 - 1, math.MaxUint8, math.MaxUint8 - 1}, + {math.MaxUint8 - 1, math.MaxUint8 - 1, math.MaxUint8 - 1}, + } + + for _, test := range tests { + if g, e := MinByte(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxUint16(t *testing.T) { + tests := []struct{ a, b, e uint16 }{ + {0, 0, 0}, + {0, 1, 1}, + {1, 0, 1}, + + {10, 10, 10}, + {10, 11, 11}, + {11, 10, 11}, + {11, 11, 11}, + + {math.MaxUint16, math.MaxUint16, math.MaxUint16}, + {math.MaxUint16, math.MaxUint16 - 1, math.MaxUint16}, + {math.MaxUint16 - 1, math.MaxUint16, math.MaxUint16}, + {math.MaxUint16 - 1, math.MaxUint16 - 1, math.MaxUint16 - 1}, + } + + for _, test := range tests { + if g, e := MaxUint16(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinUint16(t *testing.T) { + tests := []struct{ a, b, e uint16 }{ + {0, 0, 0}, + {0, 1, 0}, + {1, 0, 0}, + + {10, 10, 10}, + {10, 11, 10}, + {11, 10, 10}, + {11, 11, 11}, + + {math.MaxUint16, math.MaxUint16, math.MaxUint16}, + {math.MaxUint16, math.MaxUint16 - 1, math.MaxUint16 - 1}, + {math.MaxUint16 - 1, math.MaxUint16, math.MaxUint16 - 1}, + {math.MaxUint16 - 1, math.MaxUint16 - 1, math.MaxUint16 - 1}, + } + + for _, test := range tests { + if g, e := MinUint16(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxUint32(t *testing.T) { + tests := []struct{ a, b, e uint32 }{ + {0, 0, 0}, + {0, 1, 1}, + {1, 0, 1}, + + {10, 10, 10}, + {10, 11, 11}, + {11, 10, 11}, + {11, 11, 11}, + + {math.MaxUint32, math.MaxUint32, math.MaxUint32}, + {math.MaxUint32, math.MaxUint32 - 1, math.MaxUint32}, + {math.MaxUint32 - 1, math.MaxUint32, math.MaxUint32}, + {math.MaxUint32 - 1, math.MaxUint32 - 1, math.MaxUint32 - 1}, + } + + for _, test := range tests { + if g, e := MaxUint32(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinUint32(t *testing.T) { + tests := []struct{ a, b, e uint32 }{ + {0, 0, 0}, + {0, 1, 0}, + {1, 0, 0}, + + {10, 10, 10}, + {10, 11, 10}, + {11, 10, 10}, + {11, 11, 11}, + + {math.MaxUint32, math.MaxUint32, math.MaxUint32}, + {math.MaxUint32, math.MaxUint32 - 1, math.MaxUint32 - 1}, + {math.MaxUint32 - 1, math.MaxUint32, math.MaxUint32 - 1}, + {math.MaxUint32 - 1, math.MaxUint32 - 1, math.MaxUint32 - 1}, + } + + for _, test := range tests { + if g, e := MinUint32(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxUint64(t *testing.T) { + tests := []struct{ a, b, e uint64 }{ + {0, 0, 0}, + {0, 1, 1}, + {1, 0, 1}, + + {10, 10, 10}, + {10, 11, 11}, + {11, 10, 11}, + {11, 11, 11}, + + {math.MaxUint64, math.MaxUint64, math.MaxUint64}, + {math.MaxUint64, math.MaxUint64 - 1, math.MaxUint64}, + {math.MaxUint64 - 1, math.MaxUint64, math.MaxUint64}, + {math.MaxUint64 - 1, math.MaxUint64 - 1, math.MaxUint64 - 1}, + } + + for _, test := range tests { + if g, e := MaxUint64(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinUint64(t *testing.T) { + tests := []struct{ a, b, e uint64 }{ + {0, 0, 0}, + {0, 1, 0}, + {1, 0, 0}, + + {10, 10, 10}, + {10, 11, 10}, + {11, 10, 10}, + {11, 11, 11}, + + {math.MaxUint64, math.MaxUint64, math.MaxUint64}, + {math.MaxUint64, math.MaxUint64 - 1, math.MaxUint64 - 1}, + {math.MaxUint64 - 1, math.MaxUint64, math.MaxUint64 - 1}, + {math.MaxUint64 - 1, math.MaxUint64 - 1, math.MaxUint64 - 1}, + } + + for _, test := range tests { + if g, e := MinUint64(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxInt8(t *testing.T) { + tests := []struct{ a, b, e int8 }{ + {math.MinInt8, math.MinInt8, math.MinInt8}, + {math.MinInt8 + 1, math.MinInt8, math.MinInt8 + 1}, + {math.MinInt8, math.MinInt8 + 1, math.MinInt8 + 1}, + + {-1, -1, -1}, + {-1, 0, 0}, + {-1, 1, 1}, + + {0, -1, 0}, + {0, 0, 0}, + {0, 1, 1}, + + {1, -1, 1}, + {1, 0, 1}, + {1, 1, 1}, + + {math.MaxInt8, math.MaxInt8, math.MaxInt8}, + {math.MaxInt8 - 1, math.MaxInt8, math.MaxInt8}, + {math.MaxInt8, math.MaxInt8 - 1, math.MaxInt8}, + } + + for _, test := range tests { + if g, e := MaxInt8(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinInt8(t *testing.T) { + tests := []struct{ a, b, e int8 }{ + {math.MinInt8, math.MinInt8, math.MinInt8}, + {math.MinInt8 + 1, math.MinInt8, math.MinInt8}, + {math.MinInt8, math.MinInt8 + 1, math.MinInt8}, + + {-1, -1, -1}, + {-1, 0, -1}, + {-1, 1, -1}, + + {0, -1, -1}, + {0, 0, 0}, + {0, 1, 0}, + + {1, -1, -1}, + {1, 0, 0}, + {1, 1, 1}, + + {math.MaxInt8, math.MaxInt8, math.MaxInt8}, + {math.MaxInt8 - 1, math.MaxInt8, math.MaxInt8 - 1}, + {math.MaxInt8, math.MaxInt8 - 1, math.MaxInt8 - 1}, + } + + for _, test := range tests { + if g, e := MinInt8(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxInt16(t *testing.T) { + tests := []struct{ a, b, e int16 }{ + {math.MinInt16, math.MinInt16, math.MinInt16}, + {math.MinInt16 + 1, math.MinInt16, math.MinInt16 + 1}, + {math.MinInt16, math.MinInt16 + 1, math.MinInt16 + 1}, + + {-1, -1, -1}, + {-1, 0, 0}, + {-1, 1, 1}, + + {0, -1, 0}, + {0, 0, 0}, + {0, 1, 1}, + + {1, -1, 1}, + {1, 0, 1}, + {1, 1, 1}, + + {math.MaxInt16, math.MaxInt16, math.MaxInt16}, + {math.MaxInt16 - 1, math.MaxInt16, math.MaxInt16}, + {math.MaxInt16, math.MaxInt16 - 1, math.MaxInt16}, + } + + for _, test := range tests { + if g, e := MaxInt16(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinInt16(t *testing.T) { + tests := []struct{ a, b, e int16 }{ + {math.MinInt16, math.MinInt16, math.MinInt16}, + {math.MinInt16 + 1, math.MinInt16, math.MinInt16}, + {math.MinInt16, math.MinInt16 + 1, math.MinInt16}, + + {-1, -1, -1}, + {-1, 0, -1}, + {-1, 1, -1}, + + {0, -1, -1}, + {0, 0, 0}, + {0, 1, 0}, + + {1, -1, -1}, + {1, 0, 0}, + {1, 1, 1}, + + {math.MaxInt16, math.MaxInt16, math.MaxInt16}, + {math.MaxInt16 - 1, math.MaxInt16, math.MaxInt16 - 1}, + {math.MaxInt16, math.MaxInt16 - 1, math.MaxInt16 - 1}, + } + + for _, test := range tests { + if g, e := MinInt16(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxInt32(t *testing.T) { + tests := []struct{ a, b, e int32 }{ + {math.MinInt32, math.MinInt32, math.MinInt32}, + {math.MinInt32 + 1, math.MinInt32, math.MinInt32 + 1}, + {math.MinInt32, math.MinInt32 + 1, math.MinInt32 + 1}, + + {-1, -1, -1}, + {-1, 0, 0}, + {-1, 1, 1}, + + {0, -1, 0}, + {0, 0, 0}, + {0, 1, 1}, + + {1, -1, 1}, + {1, 0, 1}, + {1, 1, 1}, + + {math.MaxInt32, math.MaxInt32, math.MaxInt32}, + {math.MaxInt32 - 1, math.MaxInt32, math.MaxInt32}, + {math.MaxInt32, math.MaxInt32 - 1, math.MaxInt32}, + } + + for _, test := range tests { + if g, e := MaxInt32(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinInt32(t *testing.T) { + tests := []struct{ a, b, e int32 }{ + {math.MinInt32, math.MinInt32, math.MinInt32}, + {math.MinInt32 + 1, math.MinInt32, math.MinInt32}, + {math.MinInt32, math.MinInt32 + 1, math.MinInt32}, + + {-1, -1, -1}, + {-1, 0, -1}, + {-1, 1, -1}, + + {0, -1, -1}, + {0, 0, 0}, + {0, 1, 0}, + + {1, -1, -1}, + {1, 0, 0}, + {1, 1, 1}, + + {math.MaxInt32, math.MaxInt32, math.MaxInt32}, + {math.MaxInt32 - 1, math.MaxInt32, math.MaxInt32 - 1}, + {math.MaxInt32, math.MaxInt32 - 1, math.MaxInt32 - 1}, + } + + for _, test := range tests { + if g, e := MinInt32(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMaxInt64(t *testing.T) { + tests := []struct{ a, b, e int64 }{ + {math.MinInt64, math.MinInt64, math.MinInt64}, + {math.MinInt64 + 1, math.MinInt64, math.MinInt64 + 1}, + {math.MinInt64, math.MinInt64 + 1, math.MinInt64 + 1}, + + {-1, -1, -1}, + {-1, 0, 0}, + {-1, 1, 1}, + + {0, -1, 0}, + {0, 0, 0}, + {0, 1, 1}, + + {1, -1, 1}, + {1, 0, 1}, + {1, 1, 1}, + + {math.MaxInt64, math.MaxInt64, math.MaxInt64}, + {math.MaxInt64 - 1, math.MaxInt64, math.MaxInt64}, + {math.MaxInt64, math.MaxInt64 - 1, math.MaxInt64}, + } + + for _, test := range tests { + if g, e := MaxInt64(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestMinInt64(t *testing.T) { + tests := []struct{ a, b, e int64 }{ + {math.MinInt64, math.MinInt64, math.MinInt64}, + {math.MinInt64 + 1, math.MinInt64, math.MinInt64}, + {math.MinInt64, math.MinInt64 + 1, math.MinInt64}, + + {-1, -1, -1}, + {-1, 0, -1}, + {-1, 1, -1}, + + {0, -1, -1}, + {0, 0, 0}, + {0, 1, 0}, + + {1, -1, -1}, + {1, 0, 0}, + {1, 1, 1}, + + {math.MaxInt64, math.MaxInt64, math.MaxInt64}, + {math.MaxInt64 - 1, math.MaxInt64, math.MaxInt64 - 1}, + {math.MaxInt64, math.MaxInt64 - 1, math.MaxInt64 - 1}, + } + + for _, test := range tests { + if g, e := MinInt64(test.a, test.b), test.e; g != e { + t.Fatal(test.a, test.b, g, e) + } + } +} + +func TestPopCountBigInt(t *testing.T) { + const N = 1e4 + rng := rand.New(rand.NewSource(42)) + lim := big.NewInt(0) + lim.SetBit(lim, 1e3, 1) + z := big.NewInt(0) + m1 := big.NewInt(-1) + for i := 0; i < N; i++ { + z.Rand(rng, lim) + g := PopCountBigInt(z) + e := 0 + for bit := 0; bit < z.BitLen(); bit++ { + if z.Bit(bit) != 0 { + e++ + } + } + if g != e { + t.Fatal(g, e) + } + + z.Mul(z, m1) + if g := PopCountBigInt(z); g != e { + t.Fatal(g, e) + } + } +} + +func benchmarkPopCountBigInt(b *testing.B, bits int) { + lim := big.NewInt(0) + lim.SetBit(lim, bits, 1) + z := big.NewInt(0) + z.Rand(rand.New(rand.NewSource(42)), lim) + b.ResetTimer() + for i := 0; i < b.N; i++ { + PopCountBigInt(z) + } +} + +func BenchmarkPopCountBigInt1e1(b *testing.B) { + benchmarkPopCountBigInt(b, 1e1) +} + +func BenchmarkPopCountBigInt1e2(b *testing.B) { + benchmarkPopCountBigInt(b, 1e2) +} + +func BenchmarkPopCountBigInt1e3(b *testing.B) { + benchmarkPopCountBigInt(b, 1e3) +} + +func BenchmarkPopCountBigIbnt1e4(b *testing.B) { + benchmarkPopCountBigInt(b, 1e4) +} + +func BenchmarkPopCountBigInt1e5(b *testing.B) { + benchmarkPopCountBigInt(b, 1e5) +} + +func BenchmarkPopCountBigInt1e6(b *testing.B) { + benchmarkPopCountBigInt(b, 1e6) +} + +func TestToBase(t *testing.T) { + x := ToBase(big.NewInt(0), 42) + e := []int{0} + if g, e := len(x), len(e); g != e { + t.Fatal(g, e) + } + + for i, g := range x { + if e := e[i]; g != e { + t.Fatal(i, g, e) + } + } + + x = ToBase(big.NewInt(2047), 22) + e = []int{1, 5, 4} + if g, e := len(x), len(e); g != e { + t.Fatal(g, e) + } + + for i, g := range x { + if e := e[i]; g != e { + t.Fatal(i, g, e) + } + } + + x = ToBase(big.NewInt(-2047), 22) + e = []int{-1, -5, -4} + if g, e := len(x), len(e); g != e { + t.Fatal(g, e) + } + + for i, g := range x { + if e := e[i]; g != e { + t.Fatal(i, g, e) + } + } +} + +func TestBug(t *testing.T) { + if BitLenUint(MaxUint) != 64 { + t.Logf("Bug reproducible only on 64 bit architecture") + return + } + + _, err := NewFC32(MinInt, MaxInt, true) + if err == nil { + t.Fatal("Expected non nil err") + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/bits.go b/Godeps/_workspace/src/github.com/cznic/mathutil/bits.go new file mode 100644 index 00000000000..6eaa4e30546 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/bits.go @@ -0,0 +1,207 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +import ( + "math/big" +) + +// BitLenByte returns the bit width of the non zero part of n. +func BitLenByte(n byte) int { + return log2[n] + 1 +} + +// BitLenUint16 returns the bit width of the non zero part of n. +func BitLenUint16(n uint16) int { + if b := n >> 8; b != 0 { + return log2[b] + 8 + 1 + } + + return log2[n] + 1 +} + +// BitLenUint32 returns the bit width of the non zero part of n. +func BitLenUint32(n uint32) int { + if b := n >> 24; b != 0 { + return log2[b] + 24 + 1 + } + + if b := n >> 16; b != 0 { + return log2[b] + 16 + 1 + } + + if b := n >> 8; b != 0 { + return log2[b] + 8 + 1 + } + + return log2[n] + 1 +} + +// BitLen returns the bit width of the non zero part of n. +func BitLen(n int) int { // Should handle correctly [future] 64 bit Go ints + if IntBits == 64 { + return BitLenUint64(uint64(n)) + } + + if b := byte(n >> 24); b != 0 { + return log2[b] + 24 + 1 + } + + if b := byte(n >> 16); b != 0 { + return log2[b] + 16 + 1 + } + + if b := byte(n >> 8); b != 0 { + return log2[b] + 8 + 1 + } + + return log2[byte(n)] + 1 +} + +// BitLenUint returns the bit width of the non zero part of n. +func BitLenUint(n uint) int { // Should handle correctly [future] 64 bit Go uints + if IntBits == 64 { + return BitLenUint64(uint64(n)) + } + + if b := n >> 24; b != 0 { + return log2[b] + 24 + 1 + } + + if b := n >> 16; b != 0 { + return log2[b] + 16 + 1 + } + + if b := n >> 8; b != 0 { + return log2[b] + 8 + 1 + } + + return log2[n] + 1 +} + +// BitLenUint64 returns the bit width of the non zero part of n. +func BitLenUint64(n uint64) int { + if b := n >> 56; b != 0 { + return log2[b] + 56 + 1 + } + + if b := n >> 48; b != 0 { + return log2[b] + 48 + 1 + } + + if b := n >> 40; b != 0 { + return log2[b] + 40 + 1 + } + + if b := n >> 32; b != 0 { + return log2[b] + 32 + 1 + } + + if b := n >> 24; b != 0 { + return log2[b] + 24 + 1 + } + + if b := n >> 16; b != 0 { + return log2[b] + 16 + 1 + } + + if b := n >> 8; b != 0 { + return log2[b] + 8 + 1 + } + + return log2[n] + 1 +} + +// BitLenUintptr returns the bit width of the non zero part of n. +func BitLenUintptr(n uintptr) int { + if b := n >> 56; b != 0 { + return log2[b] + 56 + 1 + } + + if b := n >> 48; b != 0 { + return log2[b] + 48 + 1 + } + + if b := n >> 40; b != 0 { + return log2[b] + 40 + 1 + } + + if b := n >> 32; b != 0 { + return log2[b] + 32 + 1 + } + + if b := n >> 24; b != 0 { + return log2[b] + 24 + 1 + } + + if b := n >> 16; b != 0 { + return log2[b] + 16 + 1 + } + + if b := n >> 8; b != 0 { + return log2[b] + 8 + 1 + } + + return log2[n] + 1 +} + +// PopCountByte returns population count of n (number of bits set in n). +func PopCountByte(n byte) int { + return int(popcnt[byte(n)]) +} + +// PopCountUint16 returns population count of n (number of bits set in n). +func PopCountUint16(n uint16) int { + return int(popcnt[byte(n>>8)]) + int(popcnt[byte(n)]) +} + +// PopCountUint32 returns population count of n (number of bits set in n). +func PopCountUint32(n uint32) int { + return int(popcnt[byte(n>>24)]) + int(popcnt[byte(n>>16)]) + + int(popcnt[byte(n>>8)]) + int(popcnt[byte(n)]) +} + +// PopCount returns population count of n (number of bits set in n). +func PopCount(n int) int { // Should handle correctly [future] 64 bit Go ints + if IntBits == 64 { + return PopCountUint64(uint64(n)) + } + + return PopCountUint32(uint32(n)) +} + +// PopCountUint returns population count of n (number of bits set in n). +func PopCountUint(n uint) int { // Should handle correctly [future] 64 bit Go uints + if IntBits == 64 { + return PopCountUint64(uint64(n)) + } + + return PopCountUint32(uint32(n)) +} + +// PopCountUintptr returns population count of n (number of bits set in n). +func PopCountUintptr(n uintptr) int { + if UintPtrBits == 64 { + return PopCountUint64(uint64(n)) + } + + return PopCountUint32(uint32(n)) +} + +// PopCountUint64 returns population count of n (number of bits set in n). +func PopCountUint64(n uint64) int { + return int(popcnt[byte(n>>56)]) + int(popcnt[byte(n>>48)]) + + int(popcnt[byte(n>>40)]) + int(popcnt[byte(n>>32)]) + + int(popcnt[byte(n>>24)]) + int(popcnt[byte(n>>16)]) + + int(popcnt[byte(n>>8)]) + int(popcnt[byte(n)]) +} + +// PopCountBigInt returns population count of |n| (number of bits set in |n|). +func PopCountBigInt(n *big.Int) (r int) { + for _, v := range n.Bits() { + r += PopCountUintptr(uintptr(v)) + } + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/envelope.go b/Godeps/_workspace/src/github.com/cznic/mathutil/envelope.go new file mode 100644 index 00000000000..ff8e6012a6e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/envelope.go @@ -0,0 +1,46 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +import ( + "math" +) + +// Approximation type determines approximation methods used by e.g. Envelope. +type Approximation int + +// Specific approximation method tags +const ( + _ Approximation = iota + Linear // As named + Sinusoidal // Smooth for all derivations +) + +// Envelope is an utility for defining simple curves using a small (usually) +// set of data points. Envelope returns a value defined by x, points and +// approximation. The value of x must be in [0,1) otherwise the result is +// undefined or the function may panic. Points are interpreted as dividing the +// [0,1) interval in len(points)-1 sections, so len(points) must be > 1 or the +// function may panic. According to the left and right points closing/adjacent +// to the section the resulting value is interpolated using the chosen +// approximation method. Unsupported values of approximation are silently +// interpreted as 'Linear'. +func Envelope(x float64, points []float64, approximation Approximation) float64 { + step := 1 / float64(len(points)-1) + fslot := math.Floor(x / step) + mod := x - fslot*step + slot := int(fslot) + l, r := points[slot], points[slot+1] + rmod := mod / step + switch approximation { + case Sinusoidal: + k := (math.Sin(math.Pi*(rmod-0.5)) + 1) / 2 + return l + (r-l)*k + case Linear: + fallthrough + default: + return l + (r-l)*rmod + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example/.gitignore b/Godeps/_workspace/src/github.com/cznic/mathutil/example/.gitignore new file mode 100644 index 00000000000..ae251f36fc3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example/.gitignore @@ -0,0 +1,2 @@ +example +Makefile diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example/example.go b/Godeps/_workspace/src/github.com/cznic/mathutil/example/example.go new file mode 100644 index 00000000000..30a2d115809 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example/example.go @@ -0,0 +1,48 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +// +build ignore + +package main + +import ( + "bufio" + "flag" + "github.com/cznic/mathutil" + "log" + "math" + "os" +) + +/* + +$ # Usage e.g.: +$ go run example.go -max 1024 > mathutil.dat # generate 1kB of "random" data + +*/ +func main() { + r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + log.Fatal(err) + } + + var mflag uint64 + flag.Uint64Var(&mflag, "max", 0, "limit output to max bytes") + flag.Parse() + stdout := bufio.NewWriter(os.Stdout) + if mflag != 0 { + for i := uint64(0); i < mflag; i++ { + if err := stdout.WriteByte(byte(r.Next())); err != nil { + log.Fatal(err) + } + } + stdout.Flush() + return + } + + for stdout.WriteByte(byte(r.Next())) == nil { + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example2/.gitignore b/Godeps/_workspace/src/github.com/cznic/mathutil/example2/.gitignore new file mode 100644 index 00000000000..85c399941fb --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example2/.gitignore @@ -0,0 +1,2 @@ +example2 +Makefile diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example2/example2.go b/Godeps/_workspace/src/github.com/cznic/mathutil/example2/example2.go new file mode 100644 index 00000000000..409aeef3e28 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example2/example2.go @@ -0,0 +1,66 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +// +build ignore + +package main + +import ( + "bytes" + "github.com/cznic/mathutil" + "image" + "image/png" + "io/ioutil" + "log" + "math" + "math/rand" +) + +// $ go run example2.go # view rand.png and rnd.png by your favorite pic viewer +// +// see http://www.boallen.com/random-numbers.html +func main() { + sqr := image.Rect(0, 0, 511, 511) + r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + log.Fatal("NewFC32", err) + } + + img := image.NewGray(sqr) + for y := 0; y < 512; y++ { + for x := 0; x < 512; x++ { + if r.Next()&1 != 0 { + img.Set(x, y, image.White) + } + } + } + buf := bytes.NewBuffer(nil) + if err := png.Encode(buf, img); err != nil { + log.Fatal("Encode rnd.png ", err) + } + + if err := ioutil.WriteFile("rnd.png", buf.Bytes(), 0666); err != nil { + log.Fatal("ioutil.WriteFile/rnd.png ", err) + } + + r2 := rand.New(rand.NewSource(0)) + img = image.NewGray(sqr) + for y := 0; y < 512; y++ { + for x := 0; x < 512; x++ { + if r2.Int()&1 != 0 { + img.Set(x, y, image.White) + } + } + } + buf = bytes.NewBuffer(nil) + if err := png.Encode(buf, img); err != nil { + log.Fatal("Encode rand.png ", err) + } + + if err := ioutil.WriteFile("rand.png", buf.Bytes(), 0666); err != nil { + log.Fatal("ioutil.WriteFile/rand.png ", err) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example3/.gitignore b/Godeps/_workspace/src/github.com/cznic/mathutil/example3/.gitignore new file mode 100644 index 00000000000..474f23a0498 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example3/.gitignore @@ -0,0 +1,2 @@ +example3 +Makefile diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example3/example3.go b/Godeps/_workspace/src/github.com/cznic/mathutil/example3/example3.go new file mode 100644 index 00000000000..572d108b619 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example3/example3.go @@ -0,0 +1,43 @@ +// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// blame: jnml, labs.nic.cz + +// +build ignore + +package main + +import ( + "bufio" + "flag" + "log" + "math/rand" + "os" +) + +/* + +$ # Usage e.g.: +$ go run example3.go -max 1024 > rand.dat # generate 1kB of "random" data + +*/ +func main() { + r := rand.New(rand.NewSource(1)) + var mflag uint64 + flag.Uint64Var(&mflag, "max", 0, "limit output to max bytes") + flag.Parse() + stdout := bufio.NewWriter(os.Stdout) + if mflag != 0 { + for i := uint64(0); i < mflag; i++ { + if err := stdout.WriteByte(byte(r.Int())); err != nil { + log.Fatal(err) + } + } + stdout.Flush() + return + } + + for stdout.WriteByte(byte(r.Int())) == nil { + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example4/main.go b/Godeps/_workspace/src/github.com/cznic/mathutil/example4/main.go new file mode 100644 index 00000000000..52b35655cee --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example4/main.go @@ -0,0 +1,90 @@ +// Copyright (c) 2011 jnml. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Let QRN be the number of quadratic residues of N. Let Q be QRN/N. From a +// sorted list of primorial products < 2^32 find "record breakers". "Record +// breaker" is N with new lowest Q. +// +// There are only 49 "record breakers" < 2^32. +// +// To run the example $ go run main.go +package main + +import ( + "fmt" + "math" + "sort" + "time" + + "github.com/cznic/mathutil" + "github.com/cznic/sortutil" +) + +func main() { + pp := mathutil.PrimorialProductsUint32(0, math.MaxUint32, 32) + sort.Sort(sortutil.Uint32Slice(pp)) + var bestN, bestD uint32 = 1, 1 + order, checks := 0, 0 + var ixDirty uint32 + m := make([]byte, math.MaxUint32>>3) + for _, n := range pp { + for i := range m[:ixDirty+1] { + m[i] = 0 + } + ixDirty = 0 + checks++ + limit0 := mathutil.QScaleUint32(n, bestN, bestD) + if limit0 > math.MaxUint32 { + panic(0) + } + limit := uint32(limit0) + n64 := uint64(n) + hi := n64 >> 1 + hits := uint32(0) + check := true + fmt.Printf("\r%10d %d/%d", n, checks, len(pp)) + t0 := time.Now() + for i := uint64(0); i < hi; i++ { + sq := uint32(i * i % n64) + ix := sq >> 3 + msk := byte(1 << (sq & 7)) + if m[ix]&msk == 0 { + hits++ + if hits >= limit { + check = false + break + } + } + m[ix] |= msk + if ix > ixDirty { + ixDirty = ix + } + } + + adjPrime := ".." // Composite before + if mathutil.IsPrime(n - 1) { + adjPrime = "P." // Prime before + } + switch mathutil.IsPrime(n + 1) { + case true: + adjPrime += "P" // Prime after + case false: + adjPrime += "." // Composite after + } + + if check && mathutil.QCmpUint32(hits, n, bestN, bestD) < 0 { + order++ + d := time.Since(t0) + bestN, bestD = hits, n + q := float64(hits) / float64(n) + fmt.Printf( + "\r%2s #%03d %d %d %.2f %.2E %s %s %v\n", + adjPrime, order, n, hits, + 1/q, q, d, time.Now().Format("15:04:05"), mathutil.FactorInt(n), + ) + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/example4/results b/Godeps/_workspace/src/github.com/cznic/mathutil/example4/results new file mode 100644 index 00000000000..1642c9d85c5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/example4/results @@ -0,0 +1,55 @@ +$ time go run main.go +..P #001 2 1 2.00 5.00E-01 3us 11:13:37 +P.. #002 8 3 2.67 3.75E-01 1us 11:13:37 +P.P #003 12 4 3.00 3.33E-01 1us 11:13:37 +..P #004 16 4 4.00 2.50E-01 1us 11:13:37 +P.. #005 32 7 4.57 2.19E-01 1us 11:13:37 +P.. #006 48 8 6.00 1.67E-01 1us 11:13:37 +..P #007 96 14 6.86 1.46E-01 2us 11:13:37 +... #008 144 16 9.00 1.11E-01 2us 11:13:37 +P.P #009 240 24 10.00 1.00E-01 3us 11:13:37 +... #010 288 28 10.29 9.72E-02 4us 11:13:37 +P.. #011 480 42 11.43 8.75E-02 6us 11:13:37 +..P #012 576 48 12.00 8.33E-02 7us 11:13:37 +P.. #013 720 48 15.00 6.67E-02 8us 11:13:37 +P.. #014 1440 84 17.14 5.83E-02 15us 11:13:37 +... #015 1680 96 17.50 5.71E-02 17us 11:13:37 +P.. #016 2880 144 20.00 5.00E-02 28us 11:13:37 +... #017 3600 176 20.45 4.89E-02 35us 11:13:37 +P.. #018 5040 192 26.25 3.81E-02 50us 11:13:37 +P.. #019 10080 336 30.00 3.33E-02 96us 11:13:37 +..P #020 18480 576 32.08 3.12E-02 174us 11:13:37 +..P #021 20160 576 35.00 2.86E-02 190us 11:13:37 +... #022 25200 704 35.80 2.79E-02 237us 11:13:37 +... #023 36960 1008 36.67 2.73E-02 348us 11:13:37 +... #024 50400 1232 40.91 2.44E-02 473us 11:13:37 +P.P #025 55440 1152 48.12 2.08E-02 518us 11:13:37 +P.P #026 110880 2016 55.00 1.82E-02 1.039ms 11:13:37 +... #027 221760 3456 64.17 1.56E-02 2.056ms 11:13:37 +... #028 277200 4224 65.62 1.52E-02 2.82ms 11:13:37 +... #029 443520 6624 66.96 1.49E-02 4.179ms 11:13:37 +... #030 480480 7056 68.10 1.47E-02 4.536ms 11:13:37 +... #031 554400 7392 75.00 1.33E-02 5.217ms 11:13:37 +... #032 720720 8064 89.38 1.12E-02 6.919ms 11:13:37 +P.. #033 1441440 14112 102.14 9.79E-03 14.767ms 11:13:38 +... #034 2882880 24192 119.17 8.39E-03 28.661ms 11:13:38 +... #035 3603600 29568 121.88 8.21E-03 35.55ms 11:13:38 +... #036 5765760 46368 124.35 8.04E-03 57.798ms 11:13:38 +... #037 7207200 51744 139.29 7.18E-03 75.157ms 11:13:38 +... #038 12252240 72576 168.82 5.92E-03 147.179ms 11:13:38 +P.. #039 24504480 127008 192.94 5.18E-03 507.174ms 11:13:40 +... #040 49008960 217728 225.09 4.44E-03 1.334847s 11:13:43 +P.. #041 61261200 266112 230.21 4.34E-03 1.739597s 11:13:45 +... #042 98017920 417312 234.88 4.26E-03 2.971988s 11:13:51 +... #043 122522400 465696 263.10 3.80E-03 3.767685s 11:13:57 +P.P #044 232792560 725760 320.76 3.12E-03 7.425308s 11:14:15 +P.. #045 465585120 1270080 366.58 2.73E-03 15.18066s 11:14:50 +P.. #046 931170240 2177280 427.68 2.34E-03 34.22548s 11:16:06 +..P #047 1163962800 2661120 437.40 2.29E-03 45.038331s 11:17:10 +..P #048 1862340480 4173120 446.27 2.24E-03 1m10.288676s 11:19:26 +... #049 2327925600 4656960 499.88 2.00E-03 1m31.882756s 11:21:44 +4257792000 1679/1679 +real 11m36.548s +user 11m30.530s +sys 0m1.700s +$ diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/ff/main.go b/Godeps/_workspace/src/github.com/cznic/mathutil/ff/main.go new file mode 100644 index 00000000000..6cec57afb81 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/ff/main.go @@ -0,0 +1,83 @@ +// Copyright (c) jnml. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Factor Finder - searches for Mersenne number factors of one specific special +// form. +package main + +import ( + "flag" + "fmt" + "math/big" + "runtime" + "time" + + "github.com/cznic/mathutil" +) + +const ( + pp = 1 + pp2 = 10 +) + +var ( + _1 = big.NewInt(1) + _2 = big.NewInt(2) +) + +func main() { + runtime.GOMAXPROCS(2) + oClass := flag.Uint64("c", 2, `factor "class" number`) + oDuration := flag.Duration("d", time.Second, "duration to spend on one class") + flag.Parse() + class := *oClass + for class&1 != 0 { + class >>= 1 + } + class = mathutil.MaxUint64(class, 2) + + for { + c := time.After(*oDuration) + factor := big.NewInt(0) + factor.SetUint64(class) + exp := big.NewInt(0) + oneClass: + for { + select { + case <-c: + break oneClass + default: + } + + exp.Set(factor) + factor.Lsh(factor, 1) + factor.Add(factor, _1) + if !factor.ProbablyPrime(pp) { + continue + } + + if !exp.ProbablyPrime(pp) { + continue + } + + if mathutil.ModPowBigInt(_2, exp, factor).Cmp(_1) != 0 { + continue + } + + if !factor.ProbablyPrime(pp2) { + continue + } + + if !exp.ProbablyPrime(pp2) { + continue + } + + fmt.Printf("%d: %s | M%s (%d bits)\n", class, factor, exp, factor.BitLen()) + } + + class += 2 + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mathutil.go b/Godeps/_workspace/src/github.com/cznic/mathutil/mathutil.go new file mode 100644 index 00000000000..e8f3f5622ce --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mathutil.go @@ -0,0 +1,829 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package mathutil provides utilities supplementing the standard 'math' and +// 'math/rand' packages. +// +// Compatibility issues +// +// 2013-12-13: The following functions have been REMOVED +// +// func Uint64ToBigInt(n uint64) *big.Int +// func Uint64FromBigInt(n *big.Int) (uint64, bool) +// +// 2013-05-13: The following functions are now DEPRECATED +// +// func Uint64ToBigInt(n uint64) *big.Int +// func Uint64FromBigInt(n *big.Int) (uint64, bool) +// +// These functions will be REMOVED with Go release 1.1+1. +// +// 2013-01-21: The following functions have been REMOVED +// +// func MaxInt() int +// func MinInt() int +// func MaxUint() uint +// func UintPtrBits() int +// +// They are now replaced by untyped constants +// +// MaxInt +// MinInt +// MaxUint +// UintPtrBits +// +// Additionally one more untyped constant was added +// +// IntBits +// +// This change breaks any existing code depending on the above removed +// functions. They should have not been published in the first place, that was +// unfortunate. Instead, defining such architecture and/or implementation +// specific integer limits and bit widths as untyped constants improves +// performance and allows for static dead code elimination if it depends on +// these values. Thanks to minux for pointing it out in the mail list +// (https://groups.google.com/d/msg/golang-nuts/tlPpLW6aJw8/NT3mpToH-a4J). +// +// 2012-12-12: The following functions will be DEPRECATED with Go release +// 1.0.3+1 and REMOVED with Go release 1.0.3+2, b/c of +// http://code.google.com/p/go/source/detail?r=954a79ee3ea8 +// +// func Uint64ToBigInt(n uint64) *big.Int +// func Uint64FromBigInt(n *big.Int) (uint64, bool) +package mathutil + +import ( + "math" + "math/big" +) + +// Architecture and/or implementation specific integer limits and bit widths. +const ( + MaxInt = 1<<(IntBits-1) - 1 + MinInt = -MaxInt - 1 + MaxUint = 1<>32&1 + ^uint(0)>>16&1 + ^uint(0)>>8&1 + 3) + UintPtrBits = 1 << (^uintptr(0)>>32&1 + ^uintptr(0)>>16&1 + ^uintptr(0)>>8&1 + 3) +) + +var ( + _1 = big.NewInt(1) + _2 = big.NewInt(2) +) + +// GCDByte returns the greatest common divisor of a and b. Based on: +// http://en.wikipedia.org/wiki/Euclidean_algorithm#Implementations +func GCDByte(a, b byte) byte { + for b != 0 { + a, b = b, a%b + } + return a +} + +// GCDUint16 returns the greatest common divisor of a and b. +func GCDUint16(a, b uint16) uint16 { + for b != 0 { + a, b = b, a%b + } + return a +} + +// GCD returns the greatest common divisor of a and b. +func GCDUint32(a, b uint32) uint32 { + for b != 0 { + a, b = b, a%b + } + return a +} + +// GCD64 returns the greatest common divisor of a and b. +func GCDUint64(a, b uint64) uint64 { + for b != 0 { + a, b = b, a%b + } + return a +} + +// ISqrt returns floor(sqrt(n)). Typical run time is few hundreds of ns. +func ISqrt(n uint32) (x uint32) { + if n == 0 { + return + } + + if n >= math.MaxUint16*math.MaxUint16 { + return math.MaxUint16 + } + + var px, nx uint32 + for x = n; ; px, x = x, nx { + nx = (x + n/x) / 2 + if nx == x || nx == px { + break + } + } + return +} + +// SqrtUint64 returns floor(sqrt(n)). Typical run time is about 0.5 µs. +func SqrtUint64(n uint64) (x uint64) { + if n == 0 { + return + } + + if n >= math.MaxUint32*math.MaxUint32 { + return math.MaxUint32 + } + + var px, nx uint64 + for x = n; ; px, x = x, nx { + nx = (x + n/x) / 2 + if nx == x || nx == px { + break + } + } + return +} + +// SqrtBig returns floor(sqrt(n)). It panics on n < 0. +func SqrtBig(n *big.Int) (x *big.Int) { + switch n.Sign() { + case -1: + panic(-1) + case 0: + return big.NewInt(0) + } + + var px, nx big.Int + x = big.NewInt(0) + x.SetBit(x, n.BitLen()/2+1, 1) + for { + nx.Rsh(nx.Add(x, nx.Div(n, x)), 1) + if nx.Cmp(x) == 0 || nx.Cmp(&px) == 0 { + break + } + px.Set(x) + x.Set(&nx) + } + return +} + +// Log2Byte returns log base 2 of n. It's the same as index of the highest +// bit set in n. For n == 0 -1 is returned. +func Log2Byte(n byte) int { + return log2[n] +} + +// Log2Uint16 returns log base 2 of n. It's the same as index of the highest +// bit set in n. For n == 0 -1 is returned. +func Log2Uint16(n uint16) int { + if b := n >> 8; b != 0 { + return log2[b] + 8 + } + + return log2[n] +} + +// Log2Uint32 returns log base 2 of n. It's the same as index of the highest +// bit set in n. For n == 0 -1 is returned. +func Log2Uint32(n uint32) int { + if b := n >> 24; b != 0 { + return log2[b] + 24 + } + + if b := n >> 16; b != 0 { + return log2[b] + 16 + } + + if b := n >> 8; b != 0 { + return log2[b] + 8 + } + + return log2[n] +} + +// Log2Uint64 returns log base 2 of n. It's the same as index of the highest +// bit set in n. For n == 0 -1 is returned. +func Log2Uint64(n uint64) int { + if b := n >> 56; b != 0 { + return log2[b] + 56 + } + + if b := n >> 48; b != 0 { + return log2[b] + 48 + } + + if b := n >> 40; b != 0 { + return log2[b] + 40 + } + + if b := n >> 32; b != 0 { + return log2[b] + 32 + } + + if b := n >> 24; b != 0 { + return log2[b] + 24 + } + + if b := n >> 16; b != 0 { + return log2[b] + 16 + } + + if b := n >> 8; b != 0 { + return log2[b] + 8 + } + + return log2[n] +} + +// ModPowByte computes (b^e)%m. It panics for m == 0 || b == e == 0. +// +// See also: http://en.wikipedia.org/wiki/Modular_exponentiation#Right-to-left_binary_method +func ModPowByte(b, e, m byte) byte { + if b == 0 && e == 0 { + panic(0) + } + + if m == 1 { + return 0 + } + + r := uint16(1) + for b, m := uint16(b), uint16(m); e > 0; b, e = b*b%m, e>>1 { + if e&1 == 1 { + r = r * b % m + } + } + return byte(r) +} + +// ModPowByte computes (b^e)%m. It panics for m == 0 || b == e == 0. +func ModPowUint16(b, e, m uint16) uint16 { + if b == 0 && e == 0 { + panic(0) + } + + if m == 1 { + return 0 + } + + r := uint32(1) + for b, m := uint32(b), uint32(m); e > 0; b, e = b*b%m, e>>1 { + if e&1 == 1 { + r = r * b % m + } + } + return uint16(r) +} + +// ModPowUint32 computes (b^e)%m. It panics for m == 0 || b == e == 0. +func ModPowUint32(b, e, m uint32) uint32 { + if b == 0 && e == 0 { + panic(0) + } + + if m == 1 { + return 0 + } + + r := uint64(1) + for b, m := uint64(b), uint64(m); e > 0; b, e = b*b%m, e>>1 { + if e&1 == 1 { + r = r * b % m + } + } + return uint32(r) +} + +// ModPowUint64 computes (b^e)%m. It panics for m == 0 || b == e == 0. +func ModPowUint64(b, e, m uint64) (r uint64) { + if b == 0 && e == 0 { + panic(0) + } + + if m == 1 { + return 0 + } + + return modPowBigInt(big.NewInt(0).SetUint64(b), big.NewInt(0).SetUint64(e), big.NewInt(0).SetUint64(m)).Uint64() +} + +func modPowBigInt(b, e, m *big.Int) (r *big.Int) { + r = big.NewInt(1) + for i, n := 0, e.BitLen(); i < n; i++ { + if e.Bit(i) != 0 { + r.Mod(r.Mul(r, b), m) + } + b.Mod(b.Mul(b, b), m) + } + return +} + +// ModPowBigInt computes (b^e)%m. Returns nil for e < 0. It panics for m == 0 || b == e == 0. +func ModPowBigInt(b, e, m *big.Int) (r *big.Int) { + if b.Sign() == 0 && e.Sign() == 0 { + panic(0) + } + + if m.Cmp(_1) == 0 { + return big.NewInt(0) + } + + if e.Sign() < 0 { + return + } + + return modPowBigInt(big.NewInt(0).Set(b), big.NewInt(0).Set(e), m) +} + +var uint64ToBigIntDelta big.Int + +func init() { + uint64ToBigIntDelta.SetBit(&uint64ToBigIntDelta, 63, 1) +} + +var uintptrBits int + +func init() { + x := uint64(math.MaxUint64) + uintptrBits = BitLenUintptr(uintptr(x)) +} + +// UintptrBits returns the bit width of an uintptr at the executing machine. +func UintptrBits() int { + return uintptrBits +} + +// AddUint128_64 returns the uint128 sum of uint64 a and b. +func AddUint128_64(a, b uint64) (hi uint64, lo uint64) { + lo = a + b + if lo < a { + hi = 1 + } + return +} + +// MulUint128_64 returns the uint128 bit product of uint64 a and b. +func MulUint128_64(a, b uint64) (hi, lo uint64) { + /* + 2^(2 W) ahi bhi + 2^W alo bhi + 2^W ahi blo + alo blo + + FEDCBA98 76543210 FEDCBA98 76543210 + ---- alo*blo ---- + ---- alo*bhi ---- + ---- ahi*blo ---- + ---- ahi*bhi ---- + */ + const w = 32 + const m = 1<>w, b>>w, a&m, b&m + lo = alo * blo + mid1 := alo * bhi + mid2 := ahi * blo + c1, lo := AddUint128_64(lo, mid1<>w+mid2>>w+uint64(c1+c2)) + return +} + +// PowerizeBigInt returns (e, p) such that e is the smallest number for which p +// == b^e is greater or equal n. For n < 0 or b < 2 (0, nil) is returned. +// +// NOTE: Run time for large values of n (above about 2^1e6 ~= 1e300000) can be +// significant and/or unacceptabe. For any smaller values of n the function +// typically performs in sub second time. For "small" values of n (cca bellow +// 2^1e3 ~= 1e300) the same can be easily below 10 µs. +// +// A special (and trivial) case of b == 2 is handled separately and performs +// much faster. +func PowerizeBigInt(b, n *big.Int) (e uint32, p *big.Int) { + switch { + case b.Cmp(_2) < 0 || n.Sign() < 0: + return + case n.Sign() == 0 || n.Cmp(_1) == 0: + return 0, big.NewInt(1) + case b.Cmp(_2) == 0: + p = big.NewInt(0) + e = uint32(n.BitLen() - 1) + p.SetBit(p, int(e), 1) + if p.Cmp(n) < 0 { + p.Mul(p, _2) + e++ + } + return + } + + bw := b.BitLen() + nw := n.BitLen() + p = big.NewInt(1) + var bb, r big.Int + for { + switch p.Cmp(n) { + case -1: + x := uint32((nw - p.BitLen()) / bw) + if x == 0 { + x = 1 + } + e += x + switch x { + case 1: + p.Mul(p, b) + default: + r.Set(_1) + bb.Set(b) + e := x + for { + if e&1 != 0 { + r.Mul(&r, &bb) + } + if e >>= 1; e == 0 { + break + } + + bb.Mul(&bb, &bb) + } + p.Mul(p, &r) + } + case 0, 1: + return + } + } +} + +// PowerizeUint32BigInt returns (e, p) such that e is the smallest number for +// which p == b^e is greater or equal n. For n < 0 or b < 2 (0, nil) is +// returned. +// +// More info: see PowerizeBigInt. +func PowerizeUint32BigInt(b uint32, n *big.Int) (e uint32, p *big.Int) { + switch { + case b < 2 || n.Sign() < 0: + return + case n.Sign() == 0 || n.Cmp(_1) == 0: + return 0, big.NewInt(1) + case b == 2: + p = big.NewInt(0) + e = uint32(n.BitLen() - 1) + p.SetBit(p, int(e), 1) + if p.Cmp(n) < 0 { + p.Mul(p, _2) + e++ + } + return + } + + var bb big.Int + bb.SetInt64(int64(b)) + return PowerizeBigInt(&bb, n) +} + +/* +ProbablyPrimeUint32 returns true if n is prime or n is a pseudoprime to base a. +It implements the Miller-Rabin primality test for one specific value of 'a' and +k == 1. + +Wrt pseudocode shown at +http://en.wikipedia.org/wiki/Miller-Rabin_primality_test#Algorithm_and_running_time + + Input: n > 3, an odd integer to be tested for primality; + Input: k, a parameter that determines the accuracy of the test + Output: composite if n is composite, otherwise probably prime + write n − 1 as 2^s·d with d odd by factoring powers of 2 from n − 1 + LOOP: repeat k times: + pick a random integer a in the range [2, n − 2] + x ← a^d mod n + if x = 1 or x = n − 1 then do next LOOP + for r = 1 .. s − 1 + x ← x^2 mod n + if x = 1 then return composite + if x = n − 1 then do next LOOP + return composite + return probably prime + +... this function behaves like passing 1 for 'k' and additionaly a +fixed/non-random 'a'. Otherwise it's the same algorithm. + +See also: http://mathworld.wolfram.com/Rabin-MillerStrongPseudoprimeTest.html +*/ +func ProbablyPrimeUint32(n, a uint32) bool { + d, s := n-1, 0 + for ; d&1 == 0; d, s = d>>1, s+1 { + } + x := uint64(ModPowUint32(a, d, n)) + if x == 1 || uint32(x) == n-1 { + return true + } + + for ; s > 1; s-- { + if x = x * x % uint64(n); x == 1 { + return false + } + + if uint32(x) == n-1 { + return true + } + } + return false +} + +// ProbablyPrimeUint64_32 returns true if n is prime or n is a pseudoprime to +// base a. It implements the Miller-Rabin primality test for one specific value +// of 'a' and k == 1. See also ProbablyPrimeUint32. +func ProbablyPrimeUint64_32(n uint64, a uint32) bool { + d, s := n-1, 0 + for ; d&1 == 0; d, s = d>>1, s+1 { + } + x := ModPowUint64(uint64(a), d, n) + if x == 1 || x == n-1 { + return true + } + + bx, bn := big.NewInt(0).SetUint64(x), big.NewInt(0).SetUint64(n) + for ; s > 1; s-- { + if x = bx.Mod(bx.Mul(bx, bx), bn).Uint64(); x == 1 { + return false + } + + if x == n-1 { + return true + } + } + return false +} + +// ProbablyPrimeBigInt_32 returns true if n is prime or n is a pseudoprime to +// base a. It implements the Miller-Rabin primality test for one specific value +// of 'a' and k == 1. See also ProbablyPrimeUint32. +func ProbablyPrimeBigInt_32(n *big.Int, a uint32) bool { + var d big.Int + d.Set(n) + d.Sub(&d, _1) // d <- n-1 + s := 0 + for ; d.Bit(s) == 0; s++ { + } + nMinus1 := big.NewInt(0).Set(&d) + d.Rsh(&d, uint(s)) + + x := ModPowBigInt(big.NewInt(int64(a)), &d, n) + if x.Cmp(_1) == 0 || x.Cmp(nMinus1) == 0 { + return true + } + + for ; s > 1; s-- { + if x = x.Mod(x.Mul(x, x), n); x.Cmp(_1) == 0 { + return false + } + + if x.Cmp(nMinus1) == 0 { + return true + } + } + return false +} + +// ProbablyPrimeBigInt returns true if n is prime or n is a pseudoprime to base +// a. It implements the Miller-Rabin primality test for one specific value of +// 'a' and k == 1. See also ProbablyPrimeUint32. +func ProbablyPrimeBigInt(n, a *big.Int) bool { + var d big.Int + d.Set(n) + d.Sub(&d, _1) // d <- n-1 + s := 0 + for ; d.Bit(s) == 0; s++ { + } + nMinus1 := big.NewInt(0).Set(&d) + d.Rsh(&d, uint(s)) + + x := ModPowBigInt(a, &d, n) + if x.Cmp(_1) == 0 || x.Cmp(nMinus1) == 0 { + return true + } + + for ; s > 1; s-- { + if x = x.Mod(x.Mul(x, x), n); x.Cmp(_1) == 0 { + return false + } + + if x.Cmp(nMinus1) == 0 { + return true + } + } + return false +} + +// Max returns the larger of a and b. +func Max(a, b int) int { + if a > b { + return a + } + + return b +} + +// Min returns the smaller of a and b. +func Min(a, b int) int { + if a < b { + return a + } + + return b +} + +// UMax returns the larger of a and b. +func UMax(a, b uint) uint { + if a > b { + return a + } + + return b +} + +// UMin returns the smaller of a and b. +func UMin(a, b uint) uint { + if a < b { + return a + } + + return b +} + +// MaxByte returns the larger of a and b. +func MaxByte(a, b byte) byte { + if a > b { + return a + } + + return b +} + +// MinByte returns the smaller of a and b. +func MinByte(a, b byte) byte { + if a < b { + return a + } + + return b +} + +// MaxInt8 returns the larger of a and b. +func MaxInt8(a, b int8) int8 { + if a > b { + return a + } + + return b +} + +// MinInt8 returns the smaller of a and b. +func MinInt8(a, b int8) int8 { + if a < b { + return a + } + + return b +} + +// MaxUint16 returns the larger of a and b. +func MaxUint16(a, b uint16) uint16 { + if a > b { + return a + } + + return b +} + +// MinUint16 returns the smaller of a and b. +func MinUint16(a, b uint16) uint16 { + if a < b { + return a + } + + return b +} + +// MaxInt16 returns the larger of a and b. +func MaxInt16(a, b int16) int16 { + if a > b { + return a + } + + return b +} + +// MinInt16 returns the smaller of a and b. +func MinInt16(a, b int16) int16 { + if a < b { + return a + } + + return b +} + +// MaxUint32 returns the larger of a and b. +func MaxUint32(a, b uint32) uint32 { + if a > b { + return a + } + + return b +} + +// MinUint32 returns the smaller of a and b. +func MinUint32(a, b uint32) uint32 { + if a < b { + return a + } + + return b +} + +// MaxInt32 returns the larger of a and b. +func MaxInt32(a, b int32) int32 { + if a > b { + return a + } + + return b +} + +// MinInt32 returns the smaller of a and b. +func MinInt32(a, b int32) int32 { + if a < b { + return a + } + + return b +} + +// MaxUint64 returns the larger of a and b. +func MaxUint64(a, b uint64) uint64 { + if a > b { + return a + } + + return b +} + +// MinUint64 returns the smaller of a and b. +func MinUint64(a, b uint64) uint64 { + if a < b { + return a + } + + return b +} + +// MaxInt64 returns the larger of a and b. +func MaxInt64(a, b int64) int64 { + if a > b { + return a + } + + return b +} + +// MinInt64 returns the smaller of a and b. +func MinInt64(a, b int64) int64 { + if a < b { + return a + } + + return b +} + +// ToBase produces n in base b. For example +// +// ToBase(2047, 22) -> [1, 5, 4] +// +// 1 * 22^0 1 +// 5 * 22^1 110 +// 4 * 22^2 1936 +// ---- +// 2047 +// +// ToBase panics for bases < 2. +func ToBase(n *big.Int, b int) []int { + var nn big.Int + nn.Set(n) + if b < 2 { + panic("invalid base") + } + + k := 1 + switch nn.Sign() { + case -1: + nn.Neg(&nn) + k = -1 + case 0: + return []int{0} + } + + bb := big.NewInt(int64(b)) + var r []int + rem := big.NewInt(0) + for nn.Sign() != 0 { + nn.QuoRem(&nn, bb, rem) + r = append(r, k*int(rem.Int64())) + } + return r +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/AUTHORS b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/AUTHORS new file mode 100644 index 00000000000..0078f5f5b6d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/AUTHORS @@ -0,0 +1,11 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/CONTRIBUTORS new file mode 100644 index 00000000000..5e86f0607ce --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/CONTRIBUTORS @@ -0,0 +1,9 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/LICENSE b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/LICENSE new file mode 100644 index 00000000000..4fa2a1f4fdb --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The mersenne Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/Makefile b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/Makefile new file mode 100644 index 00000000000..a12fc5750cc --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2014 The mersenne Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +all: editor + go build + go vet + golint . + go install + make todo + +editor: + go fmt + go test -i + go test + +todo: + @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alnum:]] *.go || true + @grep -n TODO *.go || true + @grep -n FIXME *.go || true + @grep -n BUG *.go || true + +clean: + rm -f *~ diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/README b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/README new file mode 100644 index 00000000000..afa401c717e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/README @@ -0,0 +1,2 @@ +Install: $ go get github.com/cznic/mathutil/mersenne +Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/mathutil/mersenne diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/all_test.go b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/all_test.go new file mode 100644 index 00000000000..ab3900af41c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/all_test.go @@ -0,0 +1,943 @@ +// Copyright (c) 2014 The mersenne Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mersenne + +import ( + "math" + "math/big" + "math/rand" + "runtime" + "sync" + "testing" + + "github.com/cznic/mathutil" +) + +func r32() *mathutil.FC32 { + r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + panic(err) + } + + return r +} + +var ( + r64lo = big.NewInt(math.MinInt64) + r64hi = big.NewInt(math.MaxInt64) +) + +func r64() *mathutil.FCBig { + r, err := mathutil.NewFCBig(r64lo, r64hi, true) + if err != nil { + panic(err) + } + + return r +} + +func TestNew(t *testing.T) { + const N = 1e4 + data := []struct{ n, m uint32 }{ + {0, 0}, + {1, 1}, + {2, 3}, + {3, 7}, + {4, 15}, + {5, 31}, + {6, 63}, + {7, 127}, + {8, 255}, + {9, 511}, + {10, 1023}, + {11, 2047}, + {12, 4095}, + {13, 8191}, + {14, 16383}, + {15, 32767}, + {16, 65535}, + {17, 131071}, + } + + e := big.NewInt(0) + for _, v := range data { + g := New(v.n) + e.SetInt64(int64(v.m)) + if g.Cmp(e) != 0 { + t.Errorf("%d: got %s, exp %s", v.n, g, e) + } + } + + r := r32() + for i := 0; i < N; i++ { + exp := uint32(r.Next()) % 1e6 + g := New(exp) + b0 := g.BitLen() + g.Add(g, _1) + b1 := g.BitLen() + if b1-b0 != 1 { + t.Fatal(i, exp, b1, b0) + } + } +} + +func benchmarkNew(b *testing.B, max uint32) { + const N = 1 << 16 + b.StopTimer() + a := make([]uint32, N) + r := r32() + for i := range a { + a[i] = uint32(r.Next()) % max + } + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + New(a[i&(N-1)]) + } +} + +func BenchmarkNew_1e1(b *testing.B) { + benchmarkNew(b, 1e1) +} + +func BenchmarkNew_1e2(b *testing.B) { + benchmarkNew(b, 1e2) +} + +func BenchmarkNew_1e3(b *testing.B) { + benchmarkNew(b, 1e3) +} + +func BenchmarkNew_1e4(b *testing.B) { + benchmarkNew(b, 1e4) +} + +func BenchmarkNew_1e5(b *testing.B) { + benchmarkNew(b, 1e5) +} + +func BenchmarkNew_1e6(b *testing.B) { + benchmarkNew(b, 1e6) +} + +func BenchmarkNew_1e7(b *testing.B) { + benchmarkNew(b, 1e7) +} + +func BenchmarkNew_1e8(b *testing.B) { + benchmarkNew(b, 1e8) +} + +func TestHasFactorUint32(t *testing.T) { + data := []struct { + d, e uint32 + r bool + }{ + {0, 42, false}, + {1, 24, true}, + {2, 22, false}, + {3, 2, true}, + {3, 3, false}, + {3, 4, true}, + {3, 5, false}, + {3, 6, true}, + {5, 4, true}, + {5, 5, false}, + {5, 6, false}, + {5, 7, false}, + {5, 8, true}, + {5, 9, false}, + {5, 10, false}, + {5, 11, false}, + {5, 12, true}, + {7, 3, true}, + {7, 6, true}, + {7, 9, true}, + {9, 6, true}, + {9, 12, true}, + {9, 18, true}, + {11, 10, true}, + {23, 11, true}, + {89, 11, true}, + {47, 23, true}, + {193707721, 67, true}, + {13007, 929, true}, + {264248689, 500471, true}, + {112027889, 1000249, true}, + {252079759, 2000633, true}, + {222054983, 3000743, true}, + {1920355681, 4000741, true}, + {330036367, 5000551, true}, + {1020081431, 6000479, true}, + {840074281, 7000619, true}, + {624031279, 8000401, true}, + {378031207, 9000743, true}, + {380036519, 10000961, true}, + {40001447, 20000723, true}, + } + + for _, v := range data { + if g, e := HasFactorUint32(v.d, v.e), v.r; g != e { + t.Errorf("d %d e %d: got %t, exp %t", v.d, v.e, g, e) + } + } +} + +func TestHasFactorUint64(t *testing.T) { + data := []struct { + d uint64 + e uint32 + r bool + }{ + {0, 42, false}, + {1, 24, true}, + {2, 22, false}, + {3, 2, true}, + {3, 3, false}, + {3, 4, true}, + {3, 5, false}, + {3, 6, true}, + {5, 4, true}, + {5, 5, false}, + {5, 6, false}, + {5, 7, false}, + {5, 8, true}, + {5, 9, false}, + {5, 10, false}, + {5, 11, false}, + {5, 12, true}, + {7, 3, true}, + {7, 6, true}, + {7, 9, true}, + {9, 6, true}, + {9, 12, true}, + {9, 18, true}, + {11, 10, true}, + {23, 11, true}, + {89, 11, true}, + {47, 23, true}, + {193707721, 67, true}, + {13007, 929, true}, + {264248689, 500471, true}, + {112027889, 1000249, true}, + {252079759, 2000633, true}, + {222054983, 3000743, true}, + {1920355681, 4000741, true}, + {330036367, 5000551, true}, + {1020081431, 6000479, true}, + {840074281, 7000619, true}, + {624031279, 8000401, true}, + {378031207, 9000743, true}, + {380036519, 10000961, true}, + {40001447, 20000723, true}, + {1872347344039, 1000099, true}, + } + + for _, v := range data { + if g, e := HasFactorUint64(v.d, v.e), v.r; g != e { + t.Errorf("d %d e %d: got %t, exp %t", v.d, v.e, g, e) + } + } +} + +func TestHasFactorBigInt(t *testing.T) { + data := []struct { + d interface{} + e uint32 + r bool + }{ + {0, 42, false}, + {1, 24, true}, + {2, 22, false}, + {3, 2, true}, + {3, 3, false}, + {3, 4, true}, + {3, 5, false}, + {3, 6, true}, + {5, 4, true}, + {5, 5, false}, + {5, 6, false}, + {5, 7, false}, + {5, 8, true}, + {5, 9, false}, + {5, 10, false}, + {5, 11, false}, + {5, 12, true}, + {7, 3, true}, + {7, 6, true}, + {7, 9, true}, + {9, 6, true}, + {9, 12, true}, + {9, 18, true}, + {11, 10, true}, + {23, 11, true}, + {89, 11, true}, + {47, 23, true}, + {193707721, 67, true}, + {13007, 929, true}, + {264248689, 500471, true}, + {112027889, 1000249, true}, + {252079759, 2000633, true}, + {222054983, 3000743, true}, + {1920355681, 4000741, true}, + {330036367, 5000551, true}, + {1020081431, 6000479, true}, + {840074281, 7000619, true}, + {624031279, 8000401, true}, + {378031207, 9000743, true}, + {380036519, 10000961, true}, + {40001447, 20000723, true}, + {"1872347344039", 1000099, true}, + {"11502865265922183403581252152383", 100279, true}, + {"533975545077050000610542659519277030089249998649", 7293457, true}, + } + + var d big.Int + for _, v := range data { + bigInt(&d, v.d) + if g, e := HasFactorBigInt(&d, v.e), v.r; g != e { + t.Errorf("d %s e %d: got %t, exp %t", &d, v.e, g, e) + } + + if g, e := HasFactorBigInt2(&d, big.NewInt(int64(v.e))), v.r; g != e { + t.Errorf("d %s e %d: got %t, exp %t", &d, v.e, g, e) + } + } +} + +var once309 sync.Once + +func BenchmarkHasFactorUint32Rnd(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct{ d, e uint32 } + a := make([]t, N) + r := r32() + for i := range a { + a[i] = t{ + uint32(r.Next()) | 1, + uint32(r.Next()), + } + } + once309.Do(func() { b.Log("Random 32 bit factor, random 32 bit exponent\n") }) + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + HasFactorUint32(v.d, v.e) + } +} + +var once332 sync.Once + +func BenchmarkHasFactorUint64Rnd(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct { + d uint64 + e uint32 + } + a := make([]t, N) + r := r64() + for i := range a { + a[i] = t{ + uint64(r.Next().Int64()) | 1, + uint32(r.Next().Int64()), + } + } + once332.Do(func() { b.Log("Random 64 bit factor, random 32 bit exponent\n") }) + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + HasFactorUint64(v.d, v.e) + } +} + +var once358 sync.Once + +func BenchmarkHasFactorBigIntRnd_128b(b *testing.B) { + const N = 1 << 16 + b.StopTimer() + type t struct { + d *big.Int + e uint32 + } + a := make([]t, N) + r, err := mathutil.NewFCBig(_1, New(128), true) + if err != nil { + b.Fatal(err) + } + r2 := r32() + for i := range a { + dd := r.Next() + a[i] = t{ + dd.SetBit(dd, 0, 1), + uint32(r2.Next()), + } + } + once358.Do(func() { b.Log("Random 128 bit factor, random 32 bit exponent\n") }) + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v := a[i&(N-1)] + HasFactorBigInt(v.d, v.e) + } +} + +var ( + f104b, _ = big.NewInt(0).SetString( // 104 bit factor of M100279 + "11502865265922183403581252152383", + 10, + ) + f137b, _ = big.NewInt(0).SetString( // 137 bit factor of M7293457 + "533975545077050000610542659519277030089249998649", + 10, + ) +) + +var once396 sync.Once + +func BenchmarkHasFactorBigInt_104b(b *testing.B) { + b.StopTimer() + once396.Do(func() { b.Log("Verify a 104 bit factor of M100279 (16.6 bit exponent)\n") }) + runtime.GC() + var r bool + b.StartTimer() + for i := 0; i < b.N; i++ { + r = HasFactorBigInt(f104b, 100279) + } + if !r { + b.Fatal(r) + } +} + +var once412 sync.Once + +func BenchmarkHasFactorBigIntMod104b(b *testing.B) { + b.StopTimer() + once412.Do(func() { b.Log("Verify a 104 bit factor of M100279 (16.6 bit exponent) using big.Int.Mod\n") }) + runtime.GC() + m := New(100279) + var x big.Int + b.StartTimer() + for i := 0; i < b.N; i++ { + x.Mod(m, f104b) + } + if x.Cmp(_0) != 0 { + b.Fatal(x) + } +} + +var once429 sync.Once + +func BenchmarkHasFactorBigInt_137b(b *testing.B) { + b.StopTimer() + once429.Do(func() { b.Log("Verify a 137 bit factor of M7293457 (22.8 bit exponent)\n") }) + runtime.GC() + var r bool + b.StartTimer() + for i := 0; i < b.N; i++ { + r = HasFactorBigInt(f137b, 7293457) + } + if !r { + b.Fatal(r) + } +} + +var once445 sync.Once + +func BenchmarkHasFactorBigIntMod137b(b *testing.B) { + b.StopTimer() + once445.Do(func() { b.Log("Verify a 137 bit factor of M7293457 (22.8 bit exponent) using big.Int.Mod\n") }) + runtime.GC() + m := New(7293457) + var x big.Int + b.StartTimer() + for i := 0; i < b.N; i++ { + x.Mod(m, f137b) + } + if x.Cmp(_0) != 0 { + b.Fatal(x) + } +} + +func bigInt(b *big.Int, v interface{}) { + switch v := v.(type) { + case int: + b.SetInt64(int64(v)) + case string: + if _, ok := b.SetString(v, 10); !ok { + panic("bigInt: bad decimal string") + } + default: + panic("bigInt: bad v.(type)") + } +} + +func TestFromFactorBigInt(t *testing.T) { + data := []struct { + d interface{} + n uint32 + }{ + {0, 0}, + {1, 1}, + {2, 0}, + {3, 2}, + {4, 0}, + {5, 4}, + {7, 3}, + {9, 6}, + {11, 10}, + {23, 11}, + {89, 11}, + {"7432339208719", 101}, + {"198582684439", 1009}, + {"20649907789079", 1009}, + {"21624641697047", 1009}, + {"30850253615723594284324529", 1009}, + {"1134327302421596486779379019599", 1009}, + {35311753, 10009}, + {"104272300687", 10009}, + {"10409374085465521", 10009}, + {"890928517778601397463", 10009}, + {6400193, 100003}, + } + + f := func(d *big.Int, max, e uint32) { + if g := FromFactorBigInt(d, max); g != e { + t.Fatalf("%s %d %d %d", d, max, g, e) + } + } + + var d big.Int + for _, v := range data { + bigInt(&d, v.d) + switch { + case v.n > 0: + f(&d, v.n-1, 0) + default: // v.n == 0 + f(&d, 100, 0) + } + f(&d, v.n, v.n) + } +} + +var f20b = big.NewInt(200000447) // 20 bit factor of M100000223 + +func benchmarkFromFactorBigInt(b *testing.B, f *big.Int, max uint32) { + var n uint32 + for i := 0; i < b.N; i++ { + n = FromFactorBigInt(f, max) + } + if n != 0 { + b.Fatal(n) + } +} + +func BenchmarkFromFactorBigInt20b_1e1(b *testing.B) { + benchmarkFromFactorBigInt(b, f20b, 1e1) +} + +func BenchmarkFromFactorBigInt20b_1e2(b *testing.B) { + benchmarkFromFactorBigInt(b, f20b, 1e2) +} + +func BenchmarkFromFactorBigInt20b_1e3(b *testing.B) { + benchmarkFromFactorBigInt(b, f20b, 1e3) +} + +func BenchmarkFromFactorBigInt20b_1e4(b *testing.B) { + benchmarkFromFactorBigInt(b, f20b, 1e4) +} + +func BenchmarkFromFactorBigInt20b_1e5(b *testing.B) { + benchmarkFromFactorBigInt(b, f20b, 1e5) +} + +func BenchmarkFromFactorBigInt20b_1e6(b *testing.B) { + benchmarkFromFactorBigInt(b, f20b, 1e6) +} + +func BenchmarkFromFactorBigInt137b_1e1(b *testing.B) { + benchmarkFromFactorBigInt(b, f137b, 1e1) +} + +func BenchmarkFromFactorBigInt137b_1e2(b *testing.B) { + benchmarkFromFactorBigInt(b, f137b, 1e2) +} + +func BenchmarkFromFactorBigInt137b_1e3(b *testing.B) { + benchmarkFromFactorBigInt(b, f137b, 1e3) +} + +func BenchmarkFromFactorBigInt137b_1e4(b *testing.B) { + benchmarkFromFactorBigInt(b, f137b, 1e4) +} + +func BenchmarkFromFactorBigInt137b_1e5(b *testing.B) { + benchmarkFromFactorBigInt(b, f137b, 1e5) +} + +func BenchmarkFromFactorBigInt137b_1e6(b *testing.B) { + benchmarkFromFactorBigInt(b, f137b, 1e6) +} +func TestMod(t *testing.T) { + const N = 1e4 + data := []struct { + mod, n int64 + exp uint32 + }{ + {0, 0x00, 3}, + {1, 0x01, 3}, + {3, 0x03, 3}, + {0, 0x07, 3}, + {1, 0x0f, 3}, + {3, 0x1f, 3}, + {0, 0x3f, 3}, + {1, 0x7f, 3}, + {3, 0xff, 3}, + {0, 0x1ff, 3}, + } + + var mod, n big.Int + for _, v := range data { + n.SetInt64(v.n) + p := Mod(&mod, &n, v.exp) + if p != &mod { + t.Fatal(p) + } + + if g, e := mod.Int64(), v.mod; g != e { + t.Fatal(v.n, v.exp, g, e) + } + } + + f := func(in int64, exp uint32) { + n.SetInt64(in) + mod.Mod(&n, New(exp)) + e := mod.Int64() + Mod(&mod, &n, exp) + g := mod.Int64() + if g != e { + t.Fatal(in, exp, g, e) + } + } + + r32, _ := mathutil.NewFC32(1, 1e6, true) + r64, _ := mathutil.NewFCBig(_0, big.NewInt(math.MaxInt64), true) + for i := 0; i < N; i++ { + f(r64.Next().Int64(), uint32(r32.Next())) + } +} + +func benchmarkMod(b *testing.B, w, exp uint32) { + b.StopTimer() + var n, mod big.Int + n.Rand(rand.New(rand.NewSource(1)), New(w)) + n.SetBit(&n, int(w), 1) + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + Mod(&mod, &n, exp) + } +} + +func benchmarkModBig(b *testing.B, w, exp uint32) { + b.StopTimer() + var n, mod big.Int + n.Rand(rand.New(rand.NewSource(1)), New(w)) + n.SetBit(&n, int(w), 1) + runtime.GC() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + mod.Mod(&n, New(exp)) + } +} + +func BenchmarkMod_1e2(b *testing.B) { + benchmarkMod(b, 1e2+2, 1e2) +} + +func BenchmarkModBig_1e2(b *testing.B) { + benchmarkModBig(b, 1e2+2, 1e2) +} + +func BenchmarkMod_1e3(b *testing.B) { + benchmarkMod(b, 1e3+2, 1e3) +} + +func BenchmarkModBig_1e3(b *testing.B) { + benchmarkModBig(b, 1e3+2, 1e3) +} + +func BenchmarkMod_1e4(b *testing.B) { + benchmarkMod(b, 1e4+2, 1e4) +} + +func BenchmarkModBig_1e4(b *testing.B) { + benchmarkModBig(b, 1e4+2, 1e4) +} + +func BenchmarkMod_1e5(b *testing.B) { + benchmarkMod(b, 1e5+2, 1e5) +} + +func BenchmarkModBig_1e5(b *testing.B) { + benchmarkModBig(b, 1e5+2, 1e5) +} + +func BenchmarkMod_1e6(b *testing.B) { + benchmarkMod(b, 1e6+2, 1e6) +} + +func BenchmarkModBig_1e6(b *testing.B) { + benchmarkModBig(b, 1e6+2, 1e6) +} + +func BenchmarkMod_1e7(b *testing.B) { + benchmarkMod(b, 1e7+2, 1e7) +} + +func BenchmarkModBig_1e7(b *testing.B) { + benchmarkModBig(b, 1e7+2, 1e7) +} + +func BenchmarkMod_1e8(b *testing.B) { + benchmarkMod(b, 1e8+2, 1e8) +} + +func BenchmarkModBig_1e8(b *testing.B) { + benchmarkModBig(b, 1e8+2, 1e8) +} + +func BenchmarkMod_5e8(b *testing.B) { + benchmarkMod(b, 5e8+2, 5e8) +} + +func BenchmarkModBig_5e8(b *testing.B) { + benchmarkModBig(b, 5e8+2, 5e8) +} + +func TestModPow(t *testing.T) { + const N = 2e2 + data := []struct{ b, e, m, r uint32 }{ + {0, 1, 1, 0}, + {0, 2, 1, 0}, + {0, 3, 1, 0}, + + {1, 0, 1, 0}, + {1, 1, 1, 0}, + {1, 2, 1, 0}, + {1, 3, 1, 0}, + + {2, 0, 1, 0}, + {2, 1, 1, 0}, + {2, 2, 1, 0}, + {2, 3, 1, 0}, + + {2, 3, 4, 8}, + {2, 3, 5, 4}, + {2, 4, 3, 1}, + {3, 3, 3, 3}, + {3, 4, 5, 30}, + } + + f := func(b, e, m uint32, expect *big.Int) { + got := ModPow(b, e, m) + if got.Cmp(expect) != 0 { + t.Fatal(b, e, m, got, expect) + } + } + + var r big.Int + for _, v := range data { + r.SetInt64(int64(v.r)) + f(v.b, v.e, v.m, &r) + } + + rg, _ := mathutil.NewFC32(2, 1<<10, true) + var bb big.Int + for i := 0; i < N; i++ { + b, e, m := uint32(rg.Next()), uint32(rg.Next()), uint32(rg.Next()) + bb.SetInt64(int64(b)) + f(b, e, m, mathutil.ModPowBigInt(&bb, New(e), New(m))) + } +} + +func benchmarkModPow2(b *testing.B, e, m uint32) { + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + ModPow2(e, m) + } +} + +func benchmarkModPow(b *testing.B, base, e, m uint32) { + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + ModPow(base, e, m) + } +} + +func benchmarkModPowBig(b *testing.B, base, e, m uint32) { + b.StopTimer() + bb := big.NewInt(int64(base)) + ee := New(e) + mm := New(m) + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + mathutil.ModPowBigInt(bb, ee, mm) + } +} + +func BenchmarkModPow2_1e2(b *testing.B) { + benchmarkModPow2(b, 1e2, 1e2+1) +} + +func BenchmarkModPow_2_1e2(b *testing.B) { + benchmarkModPow(b, 2, 1e2, 1e2+1) +} + +func BenchmarkModPowB_2_1e2(b *testing.B) { + benchmarkModPowBig(b, 2, 1e2, 1e2+1) +} + +func BenchmarkModPow_3_1e2(b *testing.B) { + benchmarkModPow(b, 3, 1e2, 1e2+1) +} + +func BenchmarkModPowB_3_1e2(b *testing.B) { + benchmarkModPowBig(b, 3, 1e2, 1e2+1) +} + +// ---- + +func BenchmarkModPow2_1e3(b *testing.B) { + benchmarkModPow2(b, 1e3, 1e3+1) +} + +func BenchmarkModPow_2_1e3(b *testing.B) { + benchmarkModPow(b, 2, 1e3, 1e3+1) +} + +func BenchmarkModPowB_2_1e3(b *testing.B) { + benchmarkModPowBig(b, 2, 1e3, 1e3+1) +} + +func BenchmarkModPow_3_1e3(b *testing.B) { + benchmarkModPow(b, 3, 1e3, 1e3+1) +} + +func BenchmarkModPowB_3_1e3(b *testing.B) { + benchmarkModPowBig(b, 3, 1e3, 1e3+1) +} + +// ---- + +func BenchmarkModPow2_1e4(b *testing.B) { + benchmarkModPow2(b, 1e4, 1e4+1) +} + +func BenchmarkModPow_2_1e4(b *testing.B) { + benchmarkModPow(b, 2, 1e4, 1e4+1) +} + +func BenchmarkModPowB_2_1e4(b *testing.B) { + benchmarkModPowBig(b, 2, 1e4, 1e4+1) +} + +func BenchmarkModPow_3_1e4(b *testing.B) { + benchmarkModPow(b, 3, 1e4, 1e4+1) +} + +func BenchmarkModPowB_3_1e4(b *testing.B) { + benchmarkModPowBig(b, 3, 1e4, 1e4+1) +} + +// ---- + +func BenchmarkModPow2_1e5(b *testing.B) { + benchmarkModPow2(b, 1e5, 1e5+1) +} + +func BenchmarkModPow2_1e6(b *testing.B) { + benchmarkModPow2(b, 1e6, 1e6+1) +} + +func BenchmarkModPow2_1e7(b *testing.B) { + benchmarkModPow2(b, 1e7, 1e7+1) +} + +func BenchmarkModPow2_1e8(b *testing.B) { + benchmarkModPow2(b, 1e8, 1e8+1) +} + +func BenchmarkModPow2_1e9(b *testing.B) { + benchmarkModPow2(b, 1e9, 1e9+1) +} + +func TestModPow2(t *testing.T) { + const N = 1e3 + data := []struct{ e, m uint32 }{ + // e == 0 -> x == 0 + {0, 2}, + {0, 3}, + {0, 4}, + + {1, 2}, + {1, 3}, + {1, 4}, + {1, 5}, + + {2, 2}, + {2, 3}, + {2, 4}, + {2, 5}, + + {3, 2}, + {3, 3}, + {3, 4}, + {3, 5}, + {3, 6}, + {3, 7}, + {3, 8}, + {3, 9}, + + {4, 2}, + {4, 3}, + {4, 4}, + {4, 5}, + {4, 6}, + {4, 7}, + {4, 8}, + {4, 9}, + } + + var got big.Int + f := func(e, m uint32) { + x := ModPow2(e, m) + exp := ModPow(2, e, m) + got.SetInt64(0) + got.SetBit(&got, int(x), 1) + if got.Cmp(exp) != 0 { + t.Fatalf("\ne %d, m %d\ng: %s\ne: %s", e, m, &got, exp) + } + } + + for _, v := range data { + f(v.e, v.m) + } + + rg, _ := mathutil.NewFC32(2, 1<<10, true) + for i := 0; i < N; i++ { + f(uint32(rg.Next()), uint32(rg.Next())) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/mersenne.go b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/mersenne.go new file mode 100644 index 00000000000..21b6f396cf0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/mersenne/mersenne.go @@ -0,0 +1,296 @@ +// Copyright (c) 2014 The mersenne Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mersenne collects utilities related to Mersenne numbers[1] and/or some +of their properties. + +Exponent + +In this documentation the term 'exponent' refers to 'n' of a Mersenne number Mn +equal to 2^n-1. This package supports only uint32 sized exponents. New() +currently supports exponents only up to math.MaxInt32 (31 bits, up to 256 MB +required to represent such Mn in memory as a big.Int). + +Links + +Referenced from above: + [1] http://en.wikipedia.org/wiki/Mersenne_number +*/ +package mersenne + +import ( + "math" + "math/big" + + "github.com/cznic/mathutil" + "github.com/remyoudompheng/bigfft" +) + +var ( + _0 = big.NewInt(0) + _1 = big.NewInt(1) + _2 = big.NewInt(2) +) + +// Knowns list the exponent of currently (March 2012) known Mersenne primes +// exponents in order. See also: http://oeis.org/A000043 for a partial list. +var Knowns = []uint32{ + 2, // #1 + 3, // #2 + 5, // #3 + 7, // #4 + 13, // #5 + 17, // #6 + 19, // #7 + 31, // #8 + 61, // #9 + 89, // #10 + + 107, // #11 + 127, // #12 + 521, // #13 + 607, // #14 + 1279, // #15 + 2203, // #16 + 2281, // #17 + 3217, // #18 + 4253, // #19 + 4423, // #20 + + 9689, // #21 + 9941, // #22 + 11213, // #23 + 19937, // #24 + 21701, // #25 + 23209, // #26 + 44497, // #27 + 86243, // #28 + 110503, // #29 + 132049, // #30 + + 216091, // #31 + 756839, // #32 + 859433, // #33 + 1257787, // #34 + 1398269, // #35 + 2976221, // #36 + 3021377, // #37 + 6972593, // #38 + 13466917, // #39 + 20996011, // #40 + + 24036583, // #41 + 25964951, // #42 + 30402457, // #43 + 32582657, // #44 + 37156667, // #45 + 42643801, // #46 + 43112609, // #47 + 57885161, // #48 +} + +// Known maps the exponent of known Mersenne primes its ordinal number/rank. +// Ranks > 41 are currently provisional. +var Known map[uint32]int + +func init() { + Known = map[uint32]int{} + for i, v := range Knowns { + Known[v] = i + 1 + } +} + +// New returns Mn == 2^n-1 for n <= math.MaxInt32 or nil otherwise. +func New(n uint32) (m *big.Int) { + if n > math.MaxInt32 { + return + } + + m = big.NewInt(0) + return m.Sub(m.SetBit(m, int(n), 1), _1) +} + +// HasFactorUint32 returns true if d | Mn. Typical run time for a 32 bit factor +// and a 32 bit exponent is < 1 µs. +func HasFactorUint32(d, n uint32) bool { + return d == 1 || d&1 != 0 && mathutil.ModPowUint32(2, n, d) == 1 +} + +// HasFactorUint64 returns true if d | Mn. Typical run time for a 64 bit factor +// and a 32 bit exponent is < 30 µs. +func HasFactorUint64(d uint64, n uint32) bool { + return d == 1 || d&1 != 0 && mathutil.ModPowUint64(2, uint64(n), d) == 1 +} + +// HasFactorBigInt returns true if d | Mn, d > 0. Typical run time for a 128 +// bit factor and a 32 bit exponent is < 75 µs. +func HasFactorBigInt(d *big.Int, n uint32) bool { + return d.Cmp(_1) == 0 || d.Sign() > 0 && d.Bit(0) == 1 && + mathutil.ModPowBigInt(_2, big.NewInt(int64(n)), d).Cmp(_1) == 0 +} + +// HasFactorBigInt2 returns true if d | Mn, d > 0 +func HasFactorBigInt2(d, n *big.Int) bool { + return d.Cmp(_1) == 0 || d.Sign() > 0 && d.Bit(0) == 1 && + mathutil.ModPowBigInt(_2, n, d).Cmp(_1) == 0 +} + +/* +FromFactorBigInt returns n such that d | Mn if n <= max and d is odd. In other +cases zero is returned. + +It is conjectured that every odd d ∊ N divides infinitely many Mersenne numbers. +The returned n should be the exponent of smallest such Mn. + +NOTE: The computation of n from a given d performs roughly in O(n). It is +thus highly recomended to use the 'max' argument to limit the "searched" +exponent upper bound as appropriate. Otherwise the computation can take a long +time as a large factor can be a divisor of a Mn with exponent above the uint32 +limits. + +The FromFactorBigInt function is a modification of the original Will +Edgington's "reverse method", discussed here: +http://tech.groups.yahoo.com/group/primenumbers/message/15061 +*/ +func FromFactorBigInt(d *big.Int, max uint32) (n uint32) { + if d.Bit(0) == 0 { + return + } + + var m big.Int + for n < max { + m.Add(&m, d) + i := 0 + for ; m.Bit(i) == 1; i++ { + if n == math.MaxUint32 { + return 0 + } + + n++ + } + m.Rsh(&m, uint(i)) + if m.Sign() == 0 { + if n > max { + n = 0 + } + return + } + } + return 0 +} + +// Mod sets mod to n % Mexp and returns mod. It panics for exp == 0 || exp >= +// math.MaxInt32 || n < 0. +func Mod(mod, n *big.Int, exp uint32) *big.Int { + if exp == 0 || exp >= math.MaxInt32 || n.Sign() < 0 { + panic(0) + } + + m := New(exp) + mod.Set(n) + var x big.Int + for mod.BitLen() > int(exp) { + x.Set(mod) + x.Rsh(&x, uint(exp)) + mod.And(mod, m) + mod.Add(mod, &x) + } + if mod.BitLen() == int(exp) && mod.Cmp(m) == 0 { + mod.SetInt64(0) + } + return mod +} + +// ModPow2 returns x such that 2^Me % Mm == 2^x. It panics for m < 2. Typical +// run time is < 1 µs. Use instead of ModPow(2, e, m) wherever possible. +func ModPow2(e, m uint32) (x uint32) { + /* + m < 2 -> panic + e == 0 -> x == 0 + e == 1 -> x == 1 + + 2^M1 % M2 == 2^1 % 3 == 2^1 10 // 2^1, 3, 5, 7 ... +2k + 2^M1 % M3 == 2^1 % 7 == 2^1 010 // 2^1, 4, 7, ... +3k + 2^M1 % M4 == 2^1 % 15 == 2^1 0010 // 2^1, 5, 9, 13... +4k + 2^M1 % M5 == 2^1 % 31 == 2^1 00010 // 2^1, 6, 11, 16... +5k + + 2^M2 % M2 == 2^3 % 3 == 2^1 10.. // 2^3, 5, 7, 9, 11, ... +2k + 2^M2 % M3 == 2^3 % 7 == 2^0 001... // 2^3, 6, 9, 12, 15, ... +3k + 2^M2 % M4 == 2^3 % 15 == 2^3 1000 // 2^3, 7, 11, 15, 19, ... +4k + 2^M2 % M5 == 2^3 % 31 == 2^3 01000 // 2^3, 8, 13, 18, 23, ... +5k + + 2^M3 % M2 == 2^7 % 3 == 2^1 10..--.. // 2^3, 5, 7... +2k + 2^M3 % M3 == 2^7 % 7 == 2^1 010...--- // 2^1, 4, 7... +3k + 2^M3 % M4 == 2^7 % 15 == 2^3 1000.... // +4k + 2^M3 % M5 == 2^7 % 31 == 2^2 00100..... // +5k + 2^M3 % M6 == 2^7 % 63 == 2^1 000010...... // +6k + 2^M3 % M7 == 2^7 % 127 == 2^0 0000001....... + 2^M3 % M8 == 2^7 % 255 == 2^7 10000000 + 2^M3 % M9 == 2^7 % 511 == 2^7 010000000 + + 2^M4 % M2 == 2^15 % 3 == 2^1 10..--..--..--.. + 2^M4 % M3 == 2^15 % 7 == 2^0 1...---...---... + 2^M4 % M4 == 2^15 % 15 == 2^3 1000....----.... + 2^M4 % M5 == 2^15 % 31 == 2^0 1.....-----..... + 2^M4 % M6 == 2^15 % 63 == 2^3 1000......------ + 2^M4 % M7 == 2^15 % 127 == 2^1 10.......------- + 2^M4 % M8 == 2^15 % 255 == 2^7 10000000........ + 2^M4 % M9 == 2^15 % 511 == 2^6 1000000......... + */ + switch { + case m < 2: + panic(0) + case e < 2: + return e + } + + if x = mathutil.ModPowUint32(2, e, m); x == 0 { + return m - 1 + } + + return x - 1 +} + +// ModPow returns b^Me % Mm. Run time grows quickly with 'e' and/or 'm' when b +// != 2 (then ModPow2 is used). +func ModPow(b, e, m uint32) (r *big.Int) { + if m == 1 { + return big.NewInt(0) + } + + if b == 2 { + x := ModPow2(e, m) + r = big.NewInt(0) + r.SetBit(r, int(x), 1) + return + } + + bb := big.NewInt(int64(b)) + r = big.NewInt(1) + for ; e != 0; e-- { + r = bigfft.Mul(r, bb) + Mod(r, r, m) + bb = bigfft.Mul(bb, bb) + Mod(bb, bb, m) + } + return +} + +// ProbablyPrime returns true if Mn is prime or is a pseudoprime to base a. +// Note: Every Mp, prime p, is a prime or is a pseudoprime to base 2, actually +// to every base 2^i, i ∊ [1, p). In contrast - it is conjectured (w/o any +// known counterexamples) that no composite Mp, prime p, is a pseudoprime to +// base 3. +func ProbablyPrime(n, a uint32) bool { + //TODO +test, +bench + if a == 2 { + return ModPow2(n-1, n) == 0 + } + + nMinus1 := New(n) + nMinus1.Sub(nMinus1, _1) + x := ModPow(a, n-1, n) + return x.Cmp(_1) == 0 || x.Cmp(nMinus1) == 0 +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/nist-sts-2-1-1-report b/Godeps/_workspace/src/github.com/cznic/mathutil/nist-sts-2-1-1-report new file mode 100644 index 00000000000..20e686c61b7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/nist-sts-2-1-1-report @@ -0,0 +1,267 @@ +$ ./example -max 100000000 > rnd.dat +$ ./assess 1000000 + G E N E R A T O R S E L E C T I O N + ______________________________________ + + [0] Input File [1] Linear Congruential + [2] Quadratic Congruential I [3] Quadratic Congruential II + [4] Cubic Congruential [5] XOR + [6] Modular Exponentiation [7] Blum-Blum-Shub + [8] Micali-Schnorr [9] G Using SHA-1 + + Enter Choice: 0 + + + User Prescribed Input File: rnd.dat + + S T A T I S T I C A L T E S T S + _________________________________ + + [01] Frequency [02] Block Frequency + [03] Cumulative Sums [04] Runs + [05] Longest Run of Ones [06] Rank + [07] Discrete Fourier Transform [08] Nonperiodic Template Matchings + [09] Overlapping Template Matchings [10] Universal Statistical + [11] Approximate Entropy [12] Random Excursions + [13] Random Excursions Variant [14] Serial + [15] Linear Complexity + + INSTRUCTIONS + Enter 0 if you DO NOT want to apply all of the + statistical tests to each sequence and 1 if you DO. + + Enter Choice: 1 + + P a r a m e t e r A d j u s t m e n t s + ----------------------------------------- + [1] Block Frequency Test - block length(M): 128 + [2] NonOverlapping Template Test - block length(m): 9 + [3] Overlapping Template Test - block length(m): 9 + [4] Approximate Entropy Test - block length(m): 10 + [5] Serial Test - block length(m): 16 + [6] Linear Complexity Test - block length(M): 500 + + Select Test (0 to continue): 0 + + How many bitstreams? 200 + + Input File Format: + [0] ASCII - A sequence of ASCII 0's and 1's + [1] Binary - Each byte in data file contains 8 bits of data + + Select input mode: 1 + + Statistical Testing In Progress......... + + Statistical Testing Complete!!!!!!!!!!!! + +$ cat experiments/AlgorithmTesting/finalAnalysisReport.txt +------------------------------------------------------------------------------ +RESULTS FOR THE UNIFORMITY OF P-VALUES AND THE PROPORTION OF PASSING SEQUENCES +------------------------------------------------------------------------------ + generator is +------------------------------------------------------------------------------ + C1 C2 C3 C4 C5 C6 C7 C8 C9 C10 P-VALUE PROPORTION STATISTICAL TEST +------------------------------------------------------------------------------ + 28 22 17 19 15 8 24 23 19 25 0.093720 198/200 Frequency + 20 18 24 14 18 17 16 28 21 24 0.504219 199/200 BlockFrequency + 25 22 17 24 19 21 22 15 16 19 0.825505 197/200 CumulativeSums + 27 17 16 22 14 26 14 25 19 20 0.304126 199/200 CumulativeSums + 22 19 14 23 22 22 13 28 13 24 0.224821 199/200 Runs + 20 24 18 21 15 13 22 23 24 20 0.719747 197/200 LongestRun + 22 26 18 22 26 15 17 22 20 12 0.410055 199/200 Rank + 25 22 26 22 20 16 20 20 16 13 0.585209 195/200 FFT + 22 11 15 26 33 24 21 13 14 21 0.013102 197/200 NonOverlappingTemplate + 17 11 16 27 19 24 19 20 28 19 0.219006 200/200 NonOverlappingTemplate + 23 27 24 15 21 11 18 27 15 19 0.162606 197/200 NonOverlappingTemplate + 21 18 13 20 19 23 20 17 26 23 0.749884 197/200 NonOverlappingTemplate + 24 22 24 24 24 21 13 15 17 16 0.494392 196/200 NonOverlappingTemplate + 24 16 23 15 23 18 25 16 18 22 0.699313 199/200 NonOverlappingTemplate + 19 23 21 16 27 18 17 20 18 21 0.859637 198/200 NonOverlappingTemplate + 12 20 16 19 26 14 30 20 24 19 0.141256 198/200 NonOverlappingTemplate + 18 21 17 21 20 14 25 19 24 21 0.859637 198/200 NonOverlappingTemplate + 24 25 21 18 23 15 23 17 16 18 0.749884 199/200 NonOverlappingTemplate + 20 22 22 18 16 22 28 16 14 22 0.574903 198/200 NonOverlappingTemplate + 18 23 22 17 24 25 19 16 23 13 0.626709 199/200 NonOverlappingTemplate + 17 22 14 19 21 21 18 19 24 25 0.842937 198/200 NonOverlappingTemplate + 18 17 26 21 22 15 22 18 21 20 0.883171 197/200 NonOverlappingTemplate + 19 25 16 32 15 19 20 18 16 20 0.236810 199/200 NonOverlappingTemplate + 19 18 15 21 24 22 18 21 20 22 0.964295 200/200 NonOverlappingTemplate + 21 14 17 23 26 19 20 22 20 18 0.834308 196/200 NonOverlappingTemplate + 15 21 17 27 26 23 21 17 24 9 0.129620 198/200 NonOverlappingTemplate + 25 17 19 19 18 22 21 22 21 16 0.951205 196/200 NonOverlappingTemplate + 20 19 24 21 19 24 16 18 17 22 0.946308 197/200 NonOverlappingTemplate + 27 16 19 18 23 19 22 17 22 17 0.807412 197/200 NonOverlappingTemplate + 14 18 21 23 23 20 14 22 20 25 0.719747 198/200 NonOverlappingTemplate + 18 22 19 12 24 25 25 22 18 15 0.474986 198/200 NonOverlappingTemplate + 21 18 23 17 19 18 28 19 20 17 0.825505 198/200 NonOverlappingTemplate + 20 19 15 16 27 20 26 17 20 20 0.657933 198/200 NonOverlappingTemplate + 17 25 21 21 11 19 22 16 27 21 0.401199 198/200 NonOverlappingTemplate + 19 16 15 18 24 19 25 25 19 20 0.769527 199/200 NonOverlappingTemplate + 18 20 20 26 20 12 24 25 19 16 0.524101 198/200 NonOverlappingTemplate + 14 16 18 23 21 21 19 19 28 21 0.668321 197/200 NonOverlappingTemplate + 21 20 23 25 21 22 19 17 14 18 0.875539 197/200 NonOverlappingTemplate + 14 16 29 22 23 13 20 29 17 17 0.099513 197/200 NonOverlappingTemplate + 14 19 27 19 17 23 18 24 20 19 0.709558 199/200 NonOverlappingTemplate + 18 15 21 19 27 22 21 23 17 17 0.779188 198/200 NonOverlappingTemplate + 13 23 13 22 22 23 22 21 21 20 0.689019 199/200 NonOverlappingTemplate + 17 14 26 26 16 21 30 15 21 14 0.096578 199/200 NonOverlappingTemplate + 18 21 24 23 21 13 23 23 19 15 0.719747 197/200 NonOverlappingTemplate + 19 21 14 32 20 15 16 18 24 21 0.202268 199/200 NonOverlappingTemplate + 27 22 20 21 21 14 15 22 14 24 0.474986 196/200 NonOverlappingTemplate + 31 12 25 11 21 18 19 16 24 23 0.050305 197/200 NonOverlappingTemplate + 17 26 20 22 15 27 22 19 12 20 0.383827 199/200 NonOverlappingTemplate + 15 22 14 14 31 15 27 18 23 21 0.078086 194/200 NonOverlappingTemplate + 19 19 14 15 24 21 25 21 20 22 0.788728 197/200 NonOverlappingTemplate + 20 21 19 22 25 18 13 24 28 10 0.153763 195/200 NonOverlappingTemplate + 23 17 21 25 21 20 13 30 14 16 0.196920 196/200 NonOverlappingTemplate + 17 31 17 22 16 15 28 23 11 20 0.050305 197/200 NonOverlappingTemplate + 15 21 26 27 15 18 19 21 18 20 0.605916 198/200 NonOverlappingTemplate + 23 18 15 14 20 21 20 20 20 29 0.554420 200/200 NonOverlappingTemplate + 22 19 19 18 19 17 22 21 31 12 0.311542 199/200 NonOverlappingTemplate + 16 22 23 21 19 19 18 24 21 17 0.960198 197/200 NonOverlappingTemplate + 21 21 17 20 16 23 25 22 18 17 0.917870 200/200 NonOverlappingTemplate + 27 17 17 16 21 20 22 18 21 21 0.859637 197/200 NonOverlappingTemplate + 18 24 15 27 18 21 18 16 24 19 0.657933 199/200 NonOverlappingTemplate + 13 16 21 21 15 25 18 22 29 20 0.326749 198/200 NonOverlappingTemplate + 18 17 23 23 15 19 26 30 11 18 0.125927 198/200 NonOverlappingTemplate + 30 21 18 22 17 21 15 17 21 18 0.544254 195/200 NonOverlappingTemplate + 12 18 19 24 16 24 18 24 28 17 0.311542 199/200 NonOverlappingTemplate + 20 15 23 15 18 30 23 18 17 21 0.410055 196/200 NonOverlappingTemplate + 15 18 23 16 29 21 22 16 19 21 0.544254 200/200 NonOverlappingTemplate + 18 16 27 13 21 22 22 21 16 24 0.534146 199/200 NonOverlappingTemplate + 20 25 18 21 16 21 17 28 21 13 0.484646 200/200 NonOverlappingTemplate + 23 22 13 22 14 20 26 18 19 23 0.574903 197/200 NonOverlappingTemplate + 21 24 25 13 19 22 18 13 24 21 0.504219 199/200 NonOverlappingTemplate + 19 13 18 25 22 15 23 28 19 18 0.410055 195/200 NonOverlappingTemplate + 20 15 27 22 26 26 14 13 21 16 0.181557 198/200 NonOverlappingTemplate + 18 18 19 23 18 20 19 21 24 20 0.991468 200/200 NonOverlappingTemplate + 18 23 17 14 20 25 22 22 22 17 0.816537 198/200 NonOverlappingTemplate + 26 15 15 11 23 21 21 16 36 16 0.005557 196/200 NonOverlappingTemplate + 27 13 21 23 21 16 19 20 16 24 0.544254 198/200 NonOverlappingTemplate + 16 15 32 17 20 23 22 19 20 16 0.262249 200/200 NonOverlappingTemplate + 26 19 24 13 24 16 18 18 13 29 0.137282 199/200 NonOverlappingTemplate + 15 18 14 27 32 21 15 20 19 19 0.112047 198/200 NonOverlappingTemplate + 22 23 22 18 20 23 19 22 16 15 0.924076 196/200 NonOverlappingTemplate + 18 17 21 22 14 17 22 24 20 25 0.798139 199/200 NonOverlappingTemplate + 15 17 19 24 21 23 17 25 23 16 0.739918 196/200 NonOverlappingTemplate + 22 11 15 26 32 25 21 13 14 21 0.017305 197/200 NonOverlappingTemplate + 22 16 19 23 22 21 21 19 17 20 0.985788 200/200 NonOverlappingTemplate + 22 28 18 24 14 20 23 21 20 10 0.230755 198/200 NonOverlappingTemplate + 14 13 22 28 14 28 17 22 23 19 0.129620 197/200 NonOverlappingTemplate + 22 16 22 20 21 21 16 19 18 25 0.935716 198/200 NonOverlappingTemplate + 15 20 23 17 19 22 21 23 18 22 0.951205 200/200 NonOverlappingTemplate + 20 24 21 19 17 19 19 24 15 22 0.930026 198/200 NonOverlappingTemplate + 18 21 15 21 17 28 24 22 20 14 0.534146 200/200 NonOverlappingTemplate + 19 15 19 19 20 20 15 25 23 25 0.779188 198/200 NonOverlappingTemplate + 17 24 25 16 15 21 18 19 23 22 0.788728 198/200 NonOverlappingTemplate + 15 20 18 25 24 15 21 31 18 13 0.141256 200/200 NonOverlappingTemplate + 24 17 19 20 18 21 15 22 24 20 0.924076 196/200 NonOverlappingTemplate + 23 18 17 21 17 28 23 21 18 14 0.605916 197/200 NonOverlappingTemplate + 21 19 22 23 16 17 20 21 22 19 0.985788 200/200 NonOverlappingTemplate + 27 17 21 27 24 15 15 17 15 22 0.304126 199/200 NonOverlappingTemplate + 25 28 20 24 13 14 16 22 19 19 0.304126 197/200 NonOverlappingTemplate + 27 16 14 24 22 18 24 20 18 17 0.564639 196/200 NonOverlappingTemplate + 18 18 24 19 19 19 26 11 27 19 0.375313 195/200 NonOverlappingTemplate + 20 15 29 19 26 16 21 11 18 25 0.141256 197/200 NonOverlappingTemplate + 19 14 21 25 11 23 22 25 26 14 0.176657 199/200 NonOverlappingTemplate + 18 23 20 17 19 18 29 22 26 8 0.102526 199/200 NonOverlappingTemplate + 22 17 18 16 18 20 19 19 25 26 0.834308 198/200 NonOverlappingTemplate + 25 18 14 16 16 24 18 18 30 21 0.268917 198/200 NonOverlappingTemplate + 24 21 23 13 12 22 20 23 20 22 0.554420 196/200 NonOverlappingTemplate + 18 21 21 30 22 17 19 14 18 20 0.534146 197/200 NonOverlappingTemplate + 25 20 22 21 15 18 17 20 17 25 0.825505 199/200 NonOverlappingTemplate + 18 21 22 21 18 20 26 16 20 18 0.941144 197/200 NonOverlappingTemplate + 23 18 22 25 12 16 17 19 26 22 0.474986 198/200 NonOverlappingTemplate + 22 18 29 23 19 23 17 17 15 17 0.534146 198/200 NonOverlappingTemplate + 19 21 17 26 18 15 22 26 15 21 0.626709 197/200 NonOverlappingTemplate + 16 20 20 23 18 21 18 18 25 21 0.955835 199/200 NonOverlappingTemplate + 23 21 20 21 22 10 15 27 15 26 0.186566 198/200 NonOverlappingTemplate + 18 26 20 26 26 18 17 17 20 12 0.358641 198/200 NonOverlappingTemplate + 24 20 21 18 24 12 19 27 14 21 0.401199 195/200 NonOverlappingTemplate + 16 25 15 21 24 18 18 25 22 16 0.657933 199/200 NonOverlappingTemplate + 24 14 17 26 15 17 17 25 21 24 0.428095 200/200 NonOverlappingTemplate + 22 24 11 20 22 24 19 18 12 28 0.176657 196/200 NonOverlappingTemplate + 27 16 27 18 27 14 13 16 21 21 0.141256 197/200 NonOverlappingTemplate + 23 25 20 18 23 17 15 23 19 17 0.834308 196/200 NonOverlappingTemplate + 19 21 20 27 16 16 18 25 16 22 0.678686 199/200 NonOverlappingTemplate + 25 22 21 19 15 19 22 19 25 13 0.657933 197/200 NonOverlappingTemplate + 19 28 21 25 20 12 18 13 29 15 0.073417 198/200 NonOverlappingTemplate + 20 24 21 19 21 15 17 24 20 19 0.941144 198/200 NonOverlappingTemplate + 18 29 23 17 24 19 17 18 16 19 0.585209 200/200 NonOverlappingTemplate + 18 28 18 16 25 21 18 20 14 22 0.544254 198/200 NonOverlappingTemplate + 22 19 23 22 22 21 21 26 12 12 0.401199 199/200 NonOverlappingTemplate + 22 15 25 16 21 27 14 22 21 17 0.484646 199/200 NonOverlappingTemplate + 18 25 20 23 30 17 13 22 18 14 0.213309 200/200 NonOverlappingTemplate + 20 23 21 21 23 29 16 13 16 18 0.410055 199/200 NonOverlappingTemplate + 21 19 16 22 31 18 20 17 18 18 0.514124 198/200 NonOverlappingTemplate + 26 22 12 14 23 17 21 24 21 20 0.455937 197/200 NonOverlappingTemplate + 21 17 18 17 14 32 21 26 18 16 0.162606 197/200 NonOverlappingTemplate + 22 24 22 23 11 15 17 18 29 19 0.230755 198/200 NonOverlappingTemplate + 19 27 20 19 23 15 24 15 21 17 0.657933 198/200 NonOverlappingTemplate + 20 25 16 10 24 13 23 21 21 27 0.149495 200/200 NonOverlappingTemplate + 19 21 21 27 17 17 19 21 21 17 0.904708 200/200 NonOverlappingTemplate + 18 23 15 19 24 21 23 21 13 23 0.719747 198/200 NonOverlappingTemplate + 26 16 28 19 19 18 17 17 16 24 0.474986 199/200 NonOverlappingTemplate + 24 32 17 18 20 13 18 18 19 21 0.236810 195/200 NonOverlappingTemplate + 26 25 18 17 12 19 20 23 21 19 0.585209 196/200 NonOverlappingTemplate + 18 26 25 12 18 16 24 19 18 24 0.410055 199/200 NonOverlappingTemplate + 27 21 22 27 21 14 18 14 23 13 0.219006 197/200 NonOverlappingTemplate + 18 23 24 16 19 21 16 26 20 17 0.798139 199/200 NonOverlappingTemplate + 19 30 15 27 14 19 24 11 22 19 0.073417 198/200 NonOverlappingTemplate + 20 23 22 20 22 15 22 21 18 17 0.964295 198/200 NonOverlappingTemplate + 22 31 16 26 13 19 17 22 24 10 0.037566 197/200 NonOverlappingTemplate + 18 24 22 14 23 19 16 18 19 27 0.637119 197/200 NonOverlappingTemplate + 19 20 21 22 21 18 19 22 20 18 0.999438 198/200 NonOverlappingTemplate + 27 15 21 18 28 18 15 23 18 17 0.375313 195/200 NonOverlappingTemplate + 26 23 20 20 23 19 20 23 14 12 0.514124 199/200 NonOverlappingTemplate + 18 19 11 15 21 24 20 26 23 23 0.428095 198/200 NonOverlappingTemplate + 19 16 21 25 19 21 15 24 24 16 0.749884 197/200 NonOverlappingTemplate + 17 26 23 18 20 26 23 14 18 15 0.494392 198/200 NonOverlappingTemplate + 15 17 19 24 21 23 17 25 23 16 0.739918 196/200 NonOverlappingTemplate + 26 19 20 20 24 22 22 13 14 20 0.605916 198/200 OverlappingTemplate + 29 24 17 21 18 13 18 21 17 22 0.446556 196/200 Universal + 22 18 22 20 20 21 22 21 18 16 0.992952 198/200 ApproximateEntropy + 14 8 13 9 11 13 13 8 7 10 0.719747 106/106 RandomExcursions + 13 18 9 7 12 12 9 6 12 8 0.236810 104/106 RandomExcursions + 11 15 10 7 11 14 9 6 12 11 0.595549 106/106 RandomExcursions + 15 7 12 12 9 11 16 8 10 6 0.350485 106/106 RandomExcursions + 10 10 12 16 10 12 10 7 13 6 0.554420 106/106 RandomExcursions + 8 7 12 10 11 16 11 13 10 8 0.657933 106/106 RandomExcursions + 9 6 12 12 14 9 11 13 10 10 0.816537 104/106 RandomExcursions + 10 10 7 12 11 9 10 13 14 10 0.911413 105/106 RandomExcursions + 8 8 12 9 10 5 13 12 17 12 0.319084 104/106 RandomExcursionsVariant + 5 11 10 11 7 11 10 15 11 15 0.455937 104/106 RandomExcursionsVariant + 6 12 11 8 12 12 12 13 13 7 0.699313 104/106 RandomExcursionsVariant + 14 10 11 6 12 9 8 12 11 13 0.779188 104/106 RandomExcursionsVariant + 12 12 10 7 17 6 6 12 13 11 0.262249 103/106 RandomExcursionsVariant + 13 8 14 13 7 6 6 13 15 11 0.249284 102/106 RandomExcursionsVariant + 12 12 12 13 7 9 6 13 12 10 0.739918 105/106 RandomExcursionsVariant + 13 15 12 8 9 10 6 9 14 10 0.574903 106/106 RandomExcursionsVariant + 10 15 9 12 14 10 8 11 7 10 0.739918 105/106 RandomExcursionsVariant + 13 12 8 11 12 11 9 10 11 9 0.978072 103/106 RandomExcursionsVariant + 10 13 12 12 8 13 8 9 14 7 0.739918 104/106 RandomExcursionsVariant + 12 10 10 14 7 8 7 13 14 11 0.657933 106/106 RandomExcursionsVariant + 10 13 10 10 13 10 12 6 10 12 0.897763 106/106 RandomExcursionsVariant + 9 12 15 8 13 8 12 8 11 10 0.779188 106/106 RandomExcursionsVariant + 9 13 15 10 10 10 8 14 6 11 0.616305 106/106 RandomExcursionsVariant + 7 17 9 12 9 11 10 16 4 11 0.129620 106/106 RandomExcursionsVariant + 10 9 10 15 7 12 7 8 12 16 0.419021 106/106 RandomExcursionsVariant + 9 12 11 8 8 9 15 12 9 13 0.798139 106/106 RandomExcursionsVariant + 17 34 11 22 22 17 19 20 13 25 0.026057 199/200 Serial + 22 20 16 22 20 18 20 18 23 21 0.989786 199/200 Serial + 12 33 25 29 21 11 21 15 14 19 0.003996 199/200 LinearComplexity + + +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +The minimum pass rate for each statistical test with the exception of the +random excursion (variant) test is approximately = 193 for a +sample size = 200 binary sequences. + +The minimum pass rate for the random excursion (variant) test +is approximately = 101 for a sample size = 106 binary sequences. + +For further guidelines construct a probability table using the MAPLE program +provided in the addendum section of the documentation. +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +$ diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/permute.go b/Godeps/_workspace/src/github.com/cznic/mathutil/permute.go new file mode 100644 index 00000000000..bf828b694f5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/permute.go @@ -0,0 +1,39 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +import ( + "sort" +) + +// Generate the first permutation of data. +func PermutationFirst(data sort.Interface) { + sort.Sort(data) +} + +// Generate the next permutation of data if possible and return true. +// Return false if there is no more permutation left. +// Based on the algorithm described here: +// http://en.wikipedia.org/wiki/Permutation#Generation_in_lexicographic_order +func PermutationNext(data sort.Interface) bool { + var k, l int + for k = data.Len() - 2; ; k-- { // 1. + if k < 0 { + return false + } + + if data.Less(k, k+1) { + break + } + } + for l = data.Len() - 1; !data.Less(k, l); l-- { // 2. + } + data.Swap(k, l) // 3. + for i, j := k+1, data.Len()-1; i < j; i++ { // 4. + data.Swap(i, j) + j-- + } + return true +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/primes.go b/Godeps/_workspace/src/github.com/cznic/mathutil/primes.go new file mode 100644 index 00000000000..2c82eb033e9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/primes.go @@ -0,0 +1,342 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +import ( + "math" +) + +// IsPrimeUint16 returns true if n is prime. Typical run time is few ns. +func IsPrimeUint16(n uint16) bool { + return n > 0 && primes16[n-1] == 1 +} + +// NextPrimeUint16 returns first prime > n and true if successful or an +// undefined value and false if there is no next prime in the uint16 limits. +// Typical run time is few ns. +func NextPrimeUint16(n uint16) (p uint16, ok bool) { + return n + uint16(primes16[n]), n < 65521 +} + +// IsPrime returns true if n is prime. Typical run time is about 100 ns. +// +//TODO rename to IsPrimeUint32 +func IsPrime(n uint32) bool { + switch { + case n&1 == 0: + return n == 2 + case n%3 == 0: + return n == 3 + case n%5 == 0: + return n == 5 + case n%7 == 0: + return n == 7 + case n%11 == 0: + return n == 11 + case n%13 == 0: + return n == 13 + case n%17 == 0: + return n == 17 + case n%19 == 0: + return n == 19 + case n%23 == 0: + return n == 23 + case n%29 == 0: + return n == 29 + case n%31 == 0: + return n == 31 + case n%37 == 0: + return n == 37 + case n%41 == 0: + return n == 41 + case n%43 == 0: + return n == 43 + case n%47 == 0: + return n == 47 + case n%53 == 0: + return n == 53 // Benchmarked optimum + case n < 65536: + // use table data + return IsPrimeUint16(uint16(n)) + default: + mod := ModPowUint32(2, (n+1)/2, n) + if mod != 2 && mod != n-2 { + return false + } + blk := &lohi[n>>24] + lo, hi := blk.lo, blk.hi + for lo <= hi { + index := (lo + hi) >> 1 + liar := liars[index] + switch { + case n > liar: + lo = index + 1 + case n < liar: + hi = index - 1 + default: + return false + } + } + return true + } +} + +// IsPrimeUint64 returns true if n is prime. Typical run time is few tens of µs. +// +// SPRP bases: http://miller-rabin.appspot.com +func IsPrimeUint64(n uint64) bool { + switch { + case n%2 == 0: + return n == 2 + case n%3 == 0: + return n == 3 + case n%5 == 0: + return n == 5 + case n%7 == 0: + return n == 7 + case n%11 == 0: + return n == 11 + case n%13 == 0: + return n == 13 + case n%17 == 0: + return n == 17 + case n%19 == 0: + return n == 19 + case n%23 == 0: + return n == 23 + case n%29 == 0: + return n == 29 + case n%31 == 0: + return n == 31 + case n%37 == 0: + return n == 37 + case n%41 == 0: + return n == 41 + case n%43 == 0: + return n == 43 + case n%47 == 0: + return n == 47 + case n%53 == 0: + return n == 53 + case n%59 == 0: + return n == 59 + case n%61 == 0: + return n == 61 + case n%67 == 0: + return n == 67 + case n%71 == 0: + return n == 71 + case n%73 == 0: + return n == 73 + case n%79 == 0: + return n == 79 + case n%83 == 0: + return n == 83 + case n%89 == 0: + return n == 89 // Benchmarked optimum + case n <= math.MaxUint16: + return IsPrimeUint16(uint16(n)) + case n <= math.MaxUint32: + return ProbablyPrimeUint32(uint32(n), 11000544) && + ProbablyPrimeUint32(uint32(n), 31481107) + case n < 105936894253: + return ProbablyPrimeUint64_32(n, 2) && + ProbablyPrimeUint64_32(n, 1005905886) && + ProbablyPrimeUint64_32(n, 1340600841) + case n < 31858317218647: + return ProbablyPrimeUint64_32(n, 2) && + ProbablyPrimeUint64_32(n, 642735) && + ProbablyPrimeUint64_32(n, 553174392) && + ProbablyPrimeUint64_32(n, 3046413974) + case n < 3071837692357849: + return ProbablyPrimeUint64_32(n, 2) && + ProbablyPrimeUint64_32(n, 75088) && + ProbablyPrimeUint64_32(n, 642735) && + ProbablyPrimeUint64_32(n, 203659041) && + ProbablyPrimeUint64_32(n, 3613982119) + default: + return ProbablyPrimeUint64_32(n, 2) && + ProbablyPrimeUint64_32(n, 325) && + ProbablyPrimeUint64_32(n, 9375) && + ProbablyPrimeUint64_32(n, 28178) && + ProbablyPrimeUint64_32(n, 450775) && + ProbablyPrimeUint64_32(n, 9780504) && + ProbablyPrimeUint64_32(n, 1795265022) + } +} + +// NextPrime returns first prime > n and true if successful or an undefined value and false if there +// is no next prime in the uint32 limits. Typical run time is about 2 µs. +// +//TODO rename to NextPrimeUint32 +func NextPrime(n uint32) (p uint32, ok bool) { + switch { + case n < 65521: + p16, _ := NextPrimeUint16(uint16(n)) + return uint32(p16), true + case n >= math.MaxUint32-4: + return + } + + n++ + var d0, d uint32 + switch mod := n % 6; mod { + case 0: + d0, d = 1, 4 + case 1: + d = 4 + case 2, 3, 4: + d0, d = 5-mod, 2 + case 5: + d = 2 + } + + p = n + d0 + if p < n { // overflow + return + } + + for { + if IsPrime(p) { + return p, true + } + + p0 := p + p += d + if p < p0 { // overflow + break + } + + d ^= 6 + } + return +} + +// NextPrimeUint64 returns first prime > n and true if successful or an undefined value and false if there +// is no next prime in the uint64 limits. Typical run time is in hundreds of µs. +func NextPrimeUint64(n uint64) (p uint64, ok bool) { + switch { + case n < 65521: + p16, _ := NextPrimeUint16(uint16(n)) + return uint64(p16), true + case n >= 18446744073709551557: // last uint64 prime + return + } + + n++ + var d0, d uint64 + switch mod := n % 6; mod { + case 0: + d0, d = 1, 4 + case 1: + d = 4 + case 2, 3, 4: + d0, d = 5-mod, 2 + case 5: + d = 2 + } + + p = n + d0 + if p < n { // overflow + return + } + + for { + if ok = IsPrimeUint64(p); ok { + break + } + + p0 := p + p += d + if p < p0 { // overflow + break + } + + d ^= 6 + } + return +} + +// FactorTerm is one term of an integer factorization. +type FactorTerm struct { + Prime uint32 // The divisor + Power uint32 // Term == Prime^Power +} + +// FactorTerms represent a factorization of an integer +type FactorTerms []FactorTerm + +// FactorInt returns prime factorization of n > 1 or nil otherwise. +// Resulting factors are ordered by Prime. Typical run time is few µs. +func FactorInt(n uint32) (f FactorTerms) { + switch { + case n < 2: + return + case IsPrime(n): + return []FactorTerm{{n, 1}} + } + + f, w := make([]FactorTerm, 9), 0 + prime16 := uint16(0) + for { + var ok bool + if prime16, ok = NextPrimeUint16(prime16); !ok { + break + } + + prime := uint32(prime16) + if prime*prime > n { + break + } + + power := uint32(0) + for n%prime == 0 { + n /= prime + power++ + } + if power != 0 { + f[w] = FactorTerm{prime, power} + w++ + } + if n == 1 { + break + } + } + if n != 1 { + f[w] = FactorTerm{n, 1} + w++ + } + return f[:w] +} + +// PrimorialProductsUint32 returns a slice of numbers in [lo, hi] which are a +// product of max 'max' primorials. The slice is not sorted. +// +// See also: http://en.wikipedia.org/wiki/Primorial +func PrimorialProductsUint32(lo, hi, max uint32) (r []uint32) { + lo64, hi64 := int64(lo), int64(hi) + if max > 31 { // N/A + max = 31 + } + + var f func(int64, int64, uint32) + f = func(n, p int64, emax uint32) { + e := uint32(1) + for n <= hi64 && e <= emax { + n *= p + if n >= lo64 && n <= hi64 { + r = append(r, uint32(n)) + } + if n < hi64 { + p, _ := NextPrime(uint32(p)) + f(n, int64(p), e) + } + e++ + } + } + + f(1, 2, max) + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/rat.go b/Godeps/_workspace/src/github.com/cznic/mathutil/rat.go new file mode 100644 index 00000000000..91b1c6fb1ec --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/rat.go @@ -0,0 +1,27 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +// QCmpUint32 compares a/b and c/d and returns: +// +// -1 if a/b < c/d +// 0 if a/b == c/d +// +1 if a/b > c/d +// +func QCmpUint32(a, b, c, d uint32) int { + switch x, y := uint64(a)*uint64(d), uint64(b)*uint64(c); { + case x < y: + return -1 + case x == y: + return 0 + default: // x > y + return 1 + } +} + +// QScaleUint32 returns a such that a/b >= c/d. +func QScaleUint32(b, c, d uint32) (a uint64) { + return 1 + (uint64(b)*uint64(c))/uint64(d) +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/rnd.go b/Godeps/_workspace/src/github.com/cznic/mathutil/rnd.go new file mode 100644 index 00000000000..9132dc0d55e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/rnd.go @@ -0,0 +1,383 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +import ( + "fmt" + "math" + "math/big" +) + +// FC32 is a full cycle PRNG covering the 32 bit signed integer range. +// In contrast to full cycle generators shown at e.g. http://en.wikipedia.org/wiki/Full_cycle, +// this code doesn't produce values at constant delta (mod cycle length). +// The 32 bit limit is per this implementation, the algorithm used has no intrinsic limit on the cycle size. +// Properties include: +// - Adjustable limits on creation (hi, lo). +// - Positionable/randomly accessible (Pos, Seek). +// - Repeatable (deterministic). +// - Can run forward or backward (Next, Prev). +// - For a billion numbers cycle the Next/Prev PRN can be produced in cca 100-150ns. +// That's like 5-10 times slower compared to PRNs generated using the (non FC) rand package. +type FC32 struct { + cycle int64 // On average: 3 * delta / 2, (HQ: 2 * delta) + delta int64 // hi - lo + factors [][]int64 // This trades some space for hopefully a bit of speed (multiple adding vs multiplying). + lo int + mods []int // pos % set + pos int64 // Within cycle. + primes []int64 // Ordered. ∏ primes == cycle. + set []int64 // Reordered primes (magnitude order bases) according to seed. +} + +// NewFC32 returns a newly created FC32 adjusted for the closed interval [lo, hi] or an Error if any. +// If hq == true then trade some generation time for improved (pseudo)randomness. +func NewFC32(lo, hi int, hq bool) (r *FC32, err error) { + if lo > hi { + return nil, fmt.Errorf("invalid range %d > %d", lo, hi) + } + + if uint64(hi)-uint64(lo) > math.MaxUint32 { + return nil, fmt.Errorf("range out of int32 limits %d, %d", lo, hi) + } + + delta := int64(hi) - int64(lo) + // Find the primorial covering whole delta + n, set, p := int64(1), []int64{}, uint32(2) + if hq { + p++ + } + for { + set = append(set, int64(p)) + n *= int64(p) + if n > delta { + break + } + p, _ = NextPrime(p) + } + + // Adjust the set so n ∊ [delta, 2 * delta] (HQ: [delta, 3 * delta]) + // while keeping the cardinality of the set (correlates with the statistic "randomness quality") + // at max, i.e. discard atmost one member. + i := -1 // no candidate prime + if n > 2*(delta+1) { + for j, p := range set { + q := n / p + if q < delta+1 { + break + } + + i = j // mark the highest candidate prime set index + } + } + if i >= 0 { // shrink the inner cycle + n = n / set[i] + set = delete(set, i) + } + r = &FC32{ + cycle: n, + delta: delta, + factors: make([][]int64, len(set)), + lo: lo, + mods: make([]int, len(set)), + primes: set, + } + r.Seed(1) // the default seed should be always non zero + return +} + +// Cycle reports the length of the inner FCPRNG cycle. +// Cycle is atmost the double (HQ: triple) of the generator period (hi - lo + 1). +func (r *FC32) Cycle() int64 { + return r.cycle +} + +// Next returns the first PRN after Pos. +func (r *FC32) Next() int { + return r.step(1) +} + +// Pos reports the current position within the inner cycle. +func (r *FC32) Pos() int64 { + return r.pos +} + +// Prev return the first PRN before Pos. +func (r *FC32) Prev() int { + return r.step(-1) +} + +// Seed uses the provided seed value to initialize the generator to a deterministic state. +// A zero seed produces a "canonical" generator with worse randomness than for most non zero seeds. +// Still, the FC property holds for any seed value. +func (r *FC32) Seed(seed int64) { + u := uint64(seed) + r.set = mix(r.primes, &u) + n := int64(1) + for i, p := range r.set { + k := make([]int64, p) + v := int64(0) + for j := range k { + k[j] = v + v += n + } + n *= p + r.factors[i] = mix(k, &u) + } +} + +// Seek sets Pos to |pos| % Cycle. +func (r *FC32) Seek(pos int64) { //vet:ignore + if pos < 0 { + pos = -pos + } + pos %= r.cycle + r.pos = pos + for i, p := range r.set { + r.mods[i] = int(pos % p) + } +} + +func (r *FC32) step(dir int) int { + for { // avg loops per step: 3/2 (HQ: 2) + y := int64(0) + pos := r.pos + pos += int64(dir) + switch { + case pos < 0: + pos = r.cycle - 1 + case pos >= r.cycle: + pos = 0 + } + r.pos = pos + for i, mod := range r.mods { + mod += dir + p := int(r.set[i]) + switch { + case mod < 0: + mod = p - 1 + case mod >= p: + mod = 0 + } + r.mods[i] = mod + y += r.factors[i][mod] + } + if y <= r.delta { + return int(y) + r.lo + } + } +} + +func delete(set []int64, i int) (y []int64) { + for j, v := range set { + if j != i { + y = append(y, v) + } + } + return +} + +func mix(set []int64, seed *uint64) (y []int64) { + for len(set) != 0 { + *seed = rol(*seed) + i := int(*seed % uint64(len(set))) + y = append(y, set[i]) + set = delete(set, i) + } + return +} + +func rol(u uint64) (y uint64) { + y = u << 1 + if int64(u) < 0 { + y |= 1 + } + return +} + +// FCBig is a full cycle PRNG covering ranges outside of the int32 limits. +// For more info see the FC32 docs. +// Next/Prev PRN on a 1e15 cycle can be produced in about 2 µsec. +type FCBig struct { + cycle *big.Int // On average: 3 * delta / 2, (HQ: 2 * delta) + delta *big.Int // hi - lo + factors [][]*big.Int // This trades some space for hopefully a bit of speed (multiple adding vs multiplying). + lo *big.Int + mods []int // pos % set + pos *big.Int // Within cycle. + primes []int64 // Ordered. ∏ primes == cycle. + set []int64 // Reordered primes (magnitude order bases) according to seed. +} + +// NewFCBig returns a newly created FCBig adjusted for the closed interval [lo, hi] or an Error if any. +// If hq == true then trade some generation time for improved (pseudo)randomness. +func NewFCBig(lo, hi *big.Int, hq bool) (r *FCBig, err error) { + if lo.Cmp(hi) > 0 { + return nil, fmt.Errorf("invalid range %d > %d", lo, hi) + } + + delta := big.NewInt(0) + delta.Add(delta, hi).Sub(delta, lo) + + // Find the primorial covering whole delta + n, set, pp, p := big.NewInt(1), []int64{}, big.NewInt(0), uint32(2) + if hq { + p++ + } + for { + set = append(set, int64(p)) + pp.SetInt64(int64(p)) + n.Mul(n, pp) + if n.Cmp(delta) > 0 { + break + } + p, _ = NextPrime(p) + } + + // Adjust the set so n ∊ [delta, 2 * delta] (HQ: [delta, 3 * delta]) + // while keeping the cardinality of the set (correlates with the statistic "randomness quality") + // at max, i.e. discard atmost one member. + dd1 := big.NewInt(1) + dd1.Add(dd1, delta) + dd2 := big.NewInt(0) + dd2.Lsh(dd1, 1) + i := -1 // no candidate prime + if n.Cmp(dd2) > 0 { + q := big.NewInt(0) + for j, p := range set { + pp.SetInt64(p) + q.Set(n) + q.Div(q, pp) + if q.Cmp(dd1) < 0 { + break + } + + i = j // mark the highest candidate prime set index + } + } + if i >= 0 { // shrink the inner cycle + pp.SetInt64(set[i]) + n.Div(n, pp) + set = delete(set, i) + } + r = &FCBig{ + cycle: n, + delta: delta, + factors: make([][]*big.Int, len(set)), + lo: lo, + mods: make([]int, len(set)), + pos: big.NewInt(0), + primes: set, + } + r.Seed(1) // the default seed should be always non zero + return +} + +// Cycle reports the length of the inner FCPRNG cycle. +// Cycle is atmost the double (HQ: triple) of the generator period (hi - lo + 1). +func (r *FCBig) Cycle() *big.Int { + return r.cycle +} + +// Next returns the first PRN after Pos. +func (r *FCBig) Next() *big.Int { + return r.step(1) +} + +// Pos reports the current position within the inner cycle. +func (r *FCBig) Pos() *big.Int { + return r.pos +} + +// Prev return the first PRN before Pos. +func (r *FCBig) Prev() *big.Int { + return r.step(-1) +} + +// Seed uses the provided seed value to initialize the generator to a deterministic state. +// A zero seed produces a "canonical" generator with worse randomness than for most non zero seeds. +// Still, the FC property holds for any seed value. +func (r *FCBig) Seed(seed int64) { + u := uint64(seed) + r.set = mix(r.primes, &u) + n := big.NewInt(1) + v := big.NewInt(0) + pp := big.NewInt(0) + for i, p := range r.set { + k := make([]*big.Int, p) + v.SetInt64(0) + for j := range k { + k[j] = big.NewInt(0) + k[j].Set(v) + v.Add(v, n) + } + pp.SetInt64(p) + n.Mul(n, pp) + r.factors[i] = mixBig(k, &u) + } +} + +// Seek sets Pos to |pos| % Cycle. +func (r *FCBig) Seek(pos *big.Int) { + r.pos.Set(pos) + r.pos.Abs(r.pos) + r.pos.Mod(r.pos, r.cycle) + mod := big.NewInt(0) + pp := big.NewInt(0) + for i, p := range r.set { + pp.SetInt64(p) + r.mods[i] = int(mod.Mod(r.pos, pp).Int64()) + } +} + +func (r *FCBig) step(dir int) (y *big.Int) { + y = big.NewInt(0) + d := big.NewInt(int64(dir)) + for { // avg loops per step: 3/2 (HQ: 2) + r.pos.Add(r.pos, d) + switch { + case r.pos.Sign() < 0: + r.pos.Add(r.pos, r.cycle) + case r.pos.Cmp(r.cycle) >= 0: + r.pos.SetInt64(0) + } + for i, mod := range r.mods { + mod += dir + p := int(r.set[i]) + switch { + case mod < 0: + mod = p - 1 + case mod >= p: + mod = 0 + } + r.mods[i] = mod + y.Add(y, r.factors[i][mod]) + } + if y.Cmp(r.delta) <= 0 { + y.Add(y, r.lo) + return + } + y.SetInt64(0) + } +} + +func deleteBig(set []*big.Int, i int) (y []*big.Int) { + for j, v := range set { + if j != i { + y = append(y, v) + } + } + return +} + +func mixBig(set []*big.Int, seed *uint64) (y []*big.Int) { + for len(set) != 0 { + *seed = rol(*seed) + i := int(*seed % uint64(len(set))) + y = append(y, set[i]) + set = deleteBig(set, i) + } + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/tables.go b/Godeps/_workspace/src/github.com/cznic/mathutil/tables.go new file mode 100644 index 00000000000..f32952c007f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/tables.go @@ -0,0 +1,6995 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// "Static" data + +package mathutil + +var ( + // Set bits count in a byte + popcnt = [256]byte{ + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, // 0 + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, // 1 + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, // 2 + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, // 3 + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, // 4 + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, // 5 + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, // 6 + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, // 7 + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, // 8 + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, // 9 + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, // 10 + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, // 11 + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, // 12 + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, // 13 + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, // 14 + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, // 15 + } + + // Highest set bit index in a byte + log2 = [256]int{ + -1, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, // 0 + + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 1 + + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, // 2 + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, // 3 + + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, // 4 + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, // 5 + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, // 6 + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, // 7 + + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 8 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 9 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 10 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 11 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 12 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 13 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 14 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 15 + } + + // "Predivisors": 2-53 + liars = [3660]uint32{ + 31621, 42799, 49141, 49981, 65077, 65281, 80581, 83333, 88357, 90751, + 104653, 130561, 164737, 188057, 194221, 196093, 215749, 219781, 220729, 253241, + 256999, 271951, 280601, 282133, 357761, 390937, 458989, 486737, 489997, 514447, + 580337, 587861, 611701, 647089, 653333, 657901, 665281, 665333, 688213, 710533, + 721801, 722261, 738541, 741751, 742813, 745889, 769757, 818201, 838861, 873181, + 877099, 916327, 976873, 983401, 1016801, 1018921, 1053761, 1064053, 1073021, 1082401, + 1109461, 1132657, 1145257, 1168513, 1194649, 1207361, 1251949, 1252697, 1302451, 1325843, + 1357441, 1373653, 1397419, 1441091, 1493857, 1507963, 1509709, 1530787, 1584133, 1678541, + 1690501, 1730977, 1735841, 1811573, 1876393, 1969417, 1987021, 2004403, 2081713, 2163001, + 2181961, 2205967, 2261953, 2264369, 2269093, 2284453, 2304167, 2387797, 2487941, 2510569, + 2670361, 2746477, 2748023, 2757241, 2811271, 2909197, 2944261, 2976487, 3048841, 3090091, + 3116107, 3125281, 3225601, 3363121, 3375041, 3400013, 3413533, 3429037, 3539101, 3542533, + 3567481, 3568661, 3605429, 3656449, 3763801, 3828001, 3898129, 3911197, 3985921, 4072729, + 4181921, 4188889, 4209661, 4360621, 4469471, 4480477, 4513841, 4835209, 4863127, 4869313, + 4877641, 4922413, 5016191, 5044033, 5095177, 5173169, 5173601, 5176153, 5256091, 5271841, + 5284333, 5351537, 5489641, 5590621, 5672041, 5919187, 6027193, 6118141, 6140161, 6159301, + 6189121, 6226193, 6233977, 6236257, 6278533, 6334351, 6368689, 6386993, 6631549, 6658669, + 6779137, 6787327, 6836233, 6952037, 6955541, 6998881, 7017193, 7232321, 7306261, 7306561, + 7429117, 7462001, 7674967, 7725901, 7759937, 7820201, 7883731, 8036033, 8095447, 8239477, + 8384513, 8534233, 8725753, 8727391, 8902741, 9006401, 9056501, 9073513, 9131401, 9345541, + 9371251, 9439201, 9480461, 9533701, 9564169, 9567673, 9588151, 9591661, 9729301, 9774181, + 9863461, 10024561, 10084177, 10323769, 10331141, 10386241, 10425511, 10610063, 10700761, 10712857, + 10763653, 10974881, 11081459, 11115037, 11335501, 11541307, 11585293, 11592397, 11777599, 12032021, + 12096613, 12263131, 12322133, 12327121, 12599233, 12854437, 13057787, 13338371, 13446253, 13500313, + 13635289, 13694761, 13747361, 13773061, 14026897, 14154337, 14179537, 14324473, 14469841, 14671801, + 14676481, 14709241, 14794081, 14796289, 14865121, 15101893, 15139199, 15162941, 15188557, 15220951, + 15247621, 15479777, 15525241, 15603391, 15621409, 15700301, 15802681, 15976747, 15978007, 16070429, + 16132321, 16149169, 16153633, 16324001, 16349477, 16360381, 16705021, 16773121, 16822081, 16843009, + 16853077, 16879501, 16973393, 17098369, 17116837, 17134043, 17208601, 17236801, 17327773, 17375249, + 17405537, 17585969, 17870561, 18067501, 18073817, 18366937, 18443701, 18454921, 18535177, 18653353, + 18740971, 19328653, 19384289, 19404139, 19471033, 19607561, 20261251, 20417311, 20647621, 20968501, + 21042001, 21303343, 21306157, 21359521, 21397381, 21400481, 21623659, 21654533, 22075579, 22087477, + 22369621, 22591301, 22669501, 22711873, 22849481, 22953673, 23247901, 23382529, 23464033, 23577497, + 23634181, 23734901, 23828017, 23872213, 23963869, 24214051, 24356377, 25080101, 25150501, 25276421, + 25326001, 25457833, 25629913, 25696133, 25768261, 25909453, 26280073, 26377921, 26821601, 26840269, + 26877421, 26886817, 27108397, 27118601, 27219697, 27271151, 27279409, 27331921, 27380831, 27392041, + 27409541, 27491237, 27509653, 27664033, 27798461, 27808463, 28325881, 28527049, 28572961, 29111881, + 29214541, 29581501, 30022129, 30090817, 30185569, 30219757, 30295141, 30338593, 30388753, 30418957, + 30576151, 30662497, 30740417, 30881551, 30894307, 31040833, 31166803, 31436123, 31735621, 31759121, + 32091781, 32095057, 32168117, 32285041, 32497921, 32676481, 33146717, 33298337, 33600533, 33627301, + 33704101, 33872593, 34003061, 34043101, 34124641, 34540801, 34856167, 34944001, 35576599, 35703361, + 35820937, 35851037, 36291193, 36307981, 36861901, 36919681, 36974341, 37109467, 37376509, 37439201, + 37964809, 37988497, 38010307, 38046817, 38118763, 38210323, 39465091, 39512773, 39655153, 39684157, + 40165093, 40238797, 40315441, 40361197, 40629601, 40782589, 40827473, 40987201, 41121433, 41568101, + 41604109, 41642681, 41662297, 41840809, 42009217, 42485119, 42623017, 42984589, 43224397, 43363601, + 43661257, 44070841, 44314129, 44465221, 44482901, 45100177, 45175201, 45219329, 45414433, 45819541, + 45879941, 46094401, 46325029, 46386589, 46469809, 46517857, 46679761, 46860001, 47220367, 47903701, + 47918581, 48064021, 48191653, 48269761, 48316969, 48400753, 48448661, 48551161, 48563089, 49075417, + 49303801, 49411801, 49459801, 50155733, 50201089, 50443201, 50523661, 51030601, 51129781, 51302353, + 51500521, 52072021, 52119289, 52204237, 53283169, 53399449, 53656021, 53675623, 53695721, 53711113, + 54029741, 54449431, 55109401, 55176097, 55318957, 55729957, 56052361, 56420033, 56479897, 56810137, + 57762433, 58003213, 58422409, 58449847, 58509977, 58679941, 58755877, 59631211, 59840537, 59913157, + 59953741, 60155201, 60352921, 60547831, 60566431, 60581401, 60696661, 60738257, 60957361, 61201009, + 61219789, 61377109, 61832377, 62756641, 63001801, 63002501, 63065281, 63167743, 63318169, 63328469, + 63346999, 63388033, 64148717, 64605041, 64735897, 65144501, 65254393, 65301013, 65350801, 65359477, + 66096253, 67194401, 67642513, 67928221, 68102641, 68154001, 68165761, 68512867, 68621701, 68839597, + 69030901, 69128641, 69176647, 69228967, 69231061, 69485281, 69612061, 69885649, 70149631, 70463489, + 70593931, 70728121, 71079661, 71734417, 72498253, 72543547, 73562833, 73645001, 74411131, 74927161, + 75140137, 75565873, 76725091, 76745101, 77533123, 77648941, 77812153, 77817979, 78939089, 79398901, + 79411201, 79417801, 79464533, 79786523, 80142761, 80146909, 80375707, 80556337, 80687881, 80891009, + 81433591, 81954133, 82273201, 82506439, 82870517, 82929001, 83083001, 83103329, 83204801, 84164033, + 84350561, 84421081, 84487457, 84998503, 85328717, 85519337, 85823401, 86027329, 86438857, 86530621, + 86999837, 87499651, 87694261, 88256449, 88368853, 88661861, 89308771, 89784581, 90270613, 90278161, + 90341197, 90665789, 90698401, 91433281, 91659283, 92438581, 92625121, 93431521, 93541537, 93571633, + 93643201, 93677761, 93926197, 94316401, 94502701, 95451361, 95452781, 96135601, 96618397, 96791881, + 96888641, 96895441, 96904081, 96925921, 97255801, 97496449, 97796953, 97863529, 97924217, 99036001, + 99115297, 99486889, 99789673, 99898801, 100463443, 100618933, 100943201, 101152133, 101218921, 101270251, + 101276579, 101649241, 102004421, 102678031, 102690677, 102690901, 103301633, 104078857, 104524421, 104988673, + 105305443, 105919633, 106485121, 106622353, 106743073, 107360641, 107543333, 108596953, 109231229, 109437751, + 109541461, 109879837, 110135821, 110139499, 110312773, 110413333, 110717861, 111370141, 111654401, 112032001, + 112402981, 112828801, 113589601, 113605201, 113730481, 113892589, 114305441, 114329881, 114701341, 114842677, + 114910489, 115039081, 115174681, 115497901, 115804501, 115873801, 116090081, 116321617, 116617289, 116682721, + 116696161, 116998669, 117987841, 118466401, 118901521, 119092801, 119204809, 119261113, 119327041, 119558011, + 119743537, 119940853, 120296677, 120517021, 120838609, 121062001, 121374241, 121472359, 121609489, 122166307, + 122396737, 122941981, 123481777, 123671671, 123877081, 123987793, 124145473, 124630273, 124818601, 125284141, + 125686241, 125848577, 126132553, 127050067, 128079409, 128124151, 128396921, 128468957, 128665319, 128987429, + 129205781, 129256273, 129357061, 129461617, 129524669, 130556329, 130693393, 130944133, 131023201, 131567929, + 131938561, 132332201, 132338881, 132440521, 132575071, 133216381, 133302781, 133467517, 133800661, 134696801, + 134767153, 134868029, 135263269, 135296053, 135308881, 135945853, 135969401, 136043641, 136661201, 136722433, + 137415821, 137763037, 138030721, 138403981, 138828821, 139295701, 139487041, 140197051, 142525333, 142922413, + 143106133, 143168581, 145348529, 146156617, 146272901, 146659801, 146843929, 146884393, 147028001, 147287141, + 148109473, 148171769, 148910653, 149389633, 150379693, 150960239, 150988753, 151533377, 151589881, 152716537, + 152922001, 152991841, 153369061, 153589801, 153754873, 153928133, 154287451, 154513633, 154944533, 155203361, + 156114061, 156532799, 157069189, 157368661, 157405249, 157725829, 158068153, 158192317, 158397247, 158496911, + 158544401, 158895281, 160348189, 160378861, 160491329, 160587841, 160672201, 160730389, 161184013, 161216021, + 161289649, 161304001, 161423377, 162026869, 162067441, 162690481, 162771337, 162776041, 163442551, 163954561, + 164111281, 165061909, 165224321, 165938653, 166082309, 166339057, 166406561, 166827943, 167579497, 167582377, + 167692141, 167881121, 168566501, 169655641, 170640961, 170782921, 170856533, 171454321, 172116181, 172436713, + 172947529, 173401621, 174479729, 176030977, 176597821, 176609441, 176977921, 177167233, 177254533, 177693521, + 177927641, 177951973, 178837201, 178956971, 179083601, 179285137, 179820257, 180115489, 180497633, 180703451, + 181285001, 181285537, 181542601, 181647497, 182383111, 183677341, 184411567, 185653333, 186183469, 186393481, + 186983521, 187050529, 187667969, 187761241, 188516329, 188985961, 189714193, 189738361, 189941761, 190212181, + 190382161, 190913297, 191233813, 191648161, 191981609, 192346153, 192857761, 193330237, 193638337, 193949641, + 194556451, 196035001, 196049701, 196231393, 198982759, 199674721, 200143351, 200753281, 201261061, 202130197, + 202156813, 202538857, 203505697, 204280501, 204582457, 204766381, 205057561, 206304961, 206453509, 206504033, + 206529737, 207008569, 207030541, 207132481, 207477001, 207618781, 208051201, 208969223, 209246701, 209404369, + 209990881, 210592873, 210842113, 213035761, 214038533, 214110541, 214852609, 214858717, 215436241, 216821881, + 217123069, 217875571, 218603617, 218642029, 218947121, 219621781, 220531501, 220883521, 221368153, 221415781, + 221884001, 222010721, 222630193, 223449463, 223625851, 223782263, 224074369, 224136013, 224769241, 224957893, + 225853633, 226359547, 226450297, 227132641, 227444101, 227475481, 228652201, 228842209, 228988033, 229589413, + 230357761, 231383461, 231405701, 231927781, 232114433, 232460821, 232771501, 233110081, 234869009, 235426913, + 235928071, 237791143, 238001653, 238833421, 240068041, 240371713, 240694513, 240785047, 241505377, 242067841, + 242650717, 242860069, 243583201, 243955141, 244883981, 245006623, 245950561, 246099317, 246282511, 246434761, + 246658441, 247318957, 247321301, 247416101, 249582481, 250436033, 250958401, 250988173, 251528401, 251663837, + 251855893, 252853921, 253610281, 253893397, 255416897, 256831433, 257590661, 258020473, 258043229, 258234401, + 258944401, 259763093, 259765747, 260156101, 260518801, 260736341, 260963389, 261186001, 261703417, 262979501, + 263428181, 264269449, 264384469, 265020001, 265584133, 265735969, 265836161, 266790481, 266925601, 270525737, + 271272569, 271763467, 271826629, 271950829, 273361789, 273480637, 274701913, 274810241, 274919401, 275283401, + 275619961, 276018913, 276131137, 276542401, 276638321, 277787141, 278943061, 279377281, 280885153, 282253141, + 282471853, 282769771, 283900961, 284166877, 284301751, 284736091, 284834299, 285820501, 286316801, 287160301, + 287449091, 287715121, 288099001, 288117721, 288735277, 290643601, 290706781, 290953921, 291088513, 291461633, + 292153681, 292290181, 292433321, 292902481, 293346637, 293847721, 293938261, 295419097, 295743017, 297624961, + 297798961, 298212601, 299367877, 299736181, 301413001, 302635351, 304080001, 307629401, 307694323, 307972801, + 308483209, 309666361, 310474249, 310978027, 311177213, 311411629, 311655829, 311671361, 312408113, 312614021, + 314184487, 315034513, 315351521, 317137969, 317365933, 317641171, 317796119, 319053281, 319374577, 319440769, + 319726177, 320326003, 321324589, 321850849, 322469701, 322941881, 324477697, 325028089, 325352101, 325546873, + 326266051, 326405713, 326469137, 326628721, 326694301, 326695141, 327073601, 327093409, 327398009, 328302901, + 329153653, 329769721, 330198331, 330759617, 331658081, 331934989, 337135501, 337420679, 337665901, 337783981, + 338125537, 338458807, 338914369, 339195097, 339492169, 339794641, 341958121, 341994131, 343017529, 343052833, + 344201441, 344255551, 344776301, 346080391, 348989101, 349752913, 350031973, 350244577, 351058753, 351177769, + 352802803, 352932337, 353815801, 353932801, 354062809, 356604421, 356836819, 357348601, 357872971, 358416577, + 359394751, 359727073, 360145633, 360375181, 360787771, 361307521, 361312337, 362569201, 363170837, 363430637, + 364550761, 365077373, 365231401, 366487201, 366532321, 366652201, 367559501, 367632301, 368016949, 368476501, + 369667561, 371011801, 371611153, 372167101, 373012777, 373533617, 373669453, 373906513, 374346361, 374988661, + 376957153, 377192353, 377334497, 377458849, 377806687, 377869031, 378792649, 379732501, 380137633, 382304161, + 384100001, 385175113, 385319089, 387072661, 388695301, 390609941, 390612221, 391014937, 392679737, 393611653, + 394723177, 396864469, 399156661, 399302581, 399647221, 400385701, 400557109, 401100881, 403095967, 403293313, + 405739681, 405782623, 407737201, 407889161, 409302001, 409458241, 410613809, 410680357, 411618241, 411851389, + 412836689, 413138881, 413429801, 413778817, 414216461, 414368641, 415200361, 415204501, 415476343, 416964241, + 417767201, 417779909, 418044563, 418226581, 418616161, 418617281, 418667401, 419184481, 420607441, 421942951, + 422429041, 422928101, 423384001, 423465001, 424175761, 424411501, 424431541, 425967301, 426174101, 426219649, + 426770437, 426783811, 427294141, 428180191, 428758201, 429135841, 429509837, 430046857, 430381921, 430646401, + 430733701, 432227449, 434042801, 435016187, 435358657, 435993301, 436465501, 437247841, 437462101, 437597101, + 437866087, 439309261, 441354497, 441650591, 441758461, 442050577, 442181291, 442543553, 444660421, 445429693, + 446414621, 446619617, 449501761, 450807481, 450866021, 450872573, 452990401, 453366029, 453967739, 454745773, + 455198563, 457274161, 457320533, 459785089, 460251733, 460585861, 461151121, 461272267, 461329601, 462587329, + 462639409, 462701513, 464012033, 464955857, 465505633, 466290949, 466758181, 467100937, 468410113, 468950021, + 470120257, 470268137, 470644021, 471535373, 471664513, 472814413, 473581057, 474892741, 474970501, 474983881, + 475723849, 478614067, 479962009, 480668347, 481153501, 481239361, 482488393, 482824669, 482921297, 483006889, + 483029821, 483945601, 484200289, 486063001, 486902929, 487896601, 488104681, 488169289, 488585521, 488656981, + 489994201, 490950461, 491738801, 493108481, 494288677, 495909871, 496109729, 496560349, 497148599, 497285713, + 498662561, 498706651, 498905189, 500747293, 501172241, 501472333, 502686713, 504870241, 505473263, 505532773, + 505798213, 506349421, 507142567, 507323521, 508606771, 509302873, 509551201, 510925609, 511098521, 511215521, + 511611673, 512330281, 514738981, 516045197, 516259657, 516764063, 517662001, 518216201, 518548801, 521501473, + 522390109, 522758233, 523756711, 526067821, 526359289, 526686889, 528013333, 528043753, 528220117, 530630701, + 531095029, 531681281, 532126801, 532758241, 532800133, 533429881, 534782293, 535252867, 535428577, 535517581, + 536003333, 536114197, 536342419, 536870911, 540207097, 540621181, 540654409, 540680141, 542497201, 542536457, + 544861633, 545550433, 545622401, 546102481, 546117301, 546322201, 548080513, 548989561, 549308761, 550132741, + 550230409, 550635373, 550853137, 551313001, 552573793, 553027201, 554487121, 554599051, 554964001, 555321007, + 555465601, 556001377, 556069849, 556095433, 556114609, 557165209, 558235109, 558900821, 558977761, 561448487, + 562367821, 563298061, 563947141, 564298489, 564689381, 565664761, 565707061, 567358513, 567596401, 568902001, + 568967221, 569332177, 569495809, 570941881, 572123521, 572228929, 572430769, 572567353, 572936869, 573817861, + 573862021, 574998841, 575326033, 576724219, 577210181, 577352641, 577613261, 579606301, 579956653, 581618143, + 582389641, 582799951, 585261637, 586706821, 587343541, 588049001, 591242653, 591822001, 592467451, 592468777, + 593682169, 593728489, 595405201, 595590841, 597537361, 597717121, 599135767, 599945293, 600893921, 601606487, + 602379181, 604584221, 605454917, 605961049, 606872449, 607148653, 607750681, 608421637, 608917753, 609361567, + 609813781, 611097401, 611374453, 611770513, 611812321, 611817421, 612006253, 613849601, 614742241, 615361183, + 615760133, 615895897, 616280897, 617087701, 619239457, 619365121, 619480601, 620169409, 620544961, 620755537, + 621769669, 622137601, 623735953, 624303241, 624732421, 625060801, 625482001, 626717471, 627886657, 628868467, + 629134081, 630496621, 630622753, 630811513, 631767943, 631974613, 633289807, 635155291, 635291077, 635319361, + 636287653, 636337073, 636936697, 638502913, 638837761, 639305921, 639807781, 640650931, 640977373, 643036321, + 643316461, 643552909, 644004817, 644453633, 644457551, 644731357, 644900257, 645556481, 648056449, 648328801, + 651011329, 651064681, 651151801, 651514753, 652469641, 653235841, 653260633, 655264369, 657732349, 659526601, + 659846021, 660095641, 660754117, 661122881, 661207177, 662134201, 663760681, 665462081, 668498321, 670976641, + 670987021, 671716921, 672103001, 672108193, 673778827, 675260477, 676359391, 678481693, 680983817, 681019921, + 681124207, 681303241, 682528687, 683316001, 683362681, 684350833, 686059921, 687741401, 689537441, 690035713, + 690562601, 691131349, 692535637, 693456521, 694116893, 696042901, 696321949, 696998251, 697821857, 698192041, + 698819711, 702683101, 705303457, 705351583, 706728377, 707691601, 709409993, 710382401, 710617861, 710721001, + 714490481, 717096641, 717653129, 717831211, 720767521, 722955773, 724160251, 724969087, 725508241, 731276521, + 732805681, 734166217, 736668013, 739444021, 739576801, 740988151, 741182401, 741214237, 742017181, 742550401, + 744500641, 745493761, 745745461, 746331041, 747406801, 748638001, 749172821, 749640161, 750632137, 751226401, + 751705597, 752186593, 753233717, 753574537, 753594001, 754020361, 754874257, 756205633, 756271909, 756980137, + 758581651, 758687581, 758901701, 759252367, 759266621, 759638881, 762699649, 763907741, 764033999, 764240611, + 765378241, 766303693, 766823797, 770201221, 770909107, 770937931, 771043201, 771337891, 772495777, 773131927, + 773807401, 775368901, 775896181, 776443769, 777218989, 781471001, 782823281, 784450393, 784777393, 784783477, + 784966297, 787085857, 787209277, 788046901, 788931361, 789082001, 790453049, 791118043, 792144161, 792145729, + 794201333, 794399041, 794937601, 795064909, 796072003, 796200901, 796560703, 797418997, 797834017, 799162561, + 799630753, 799898833, 799916101, 801093011, 801227269, 801866647, 804978721, 805505957, 805771501, 807115753, + 807218413, 808214161, 809790881, 810023881, 810455101, 811110301, 811478533, 811607777, 811730923, 815430533, + 815796413, 816024161, 816215401, 816549121, 817832329, 818401321, 819466201, 819743233, 822018961, 822531841, + 824389441, 826004467, 829512001, 830664451, 831933901, 832048447, 832127489, 832169857, 833610751, 837766217, + 839268139, 839280691, 839908217, 840749761, 841217653, 841660961, 842785841, 842824981, 842960981, 843161887, + 844545271, 845376533, 846961321, 848090377, 848755969, 849548671, 852432769, 854094781, 854868257, 855734401, + 857100421, 857902861, 858687103, 859096477, 860334301, 862082677, 862678081, 863196181, 863609113, 863984881, + 865242841, 867022747, 867110501, 867638201, 868088341, 868111597, 868691401, 870985223, 871157233, 871195561, + 871908481, 876850801, 877542481, 878492941, 878940833, 879995689, 880870513, 880922657, 883276549, 884304037, + 884952001, 886180429, 887795221, 888868441, 892740853, 893692819, 894264337, 896901461, 897087361, 897283213, + 899019353, 900736411, 901848301, 902566501, 903108821, 903390643, 905040953, 907378669, 907670501, 907711561, + 908005249, 910202509, 910867481, 911484421, 914348737, 914906539, 920375821, 920696653, 921858631, 922845241, + 923437213, 926756881, 927106561, 927877001, 929159941, 930530701, 932148253, 933729421, 935794081, 936421141, + 937675393, 938376181, 939947009, 940123801, 941056273, 941734657, 943271569, 944832533, 946034057, 946787377, + 947878081, 949317217, 949697233, 952893881, 954924013, 957600541, 957631249, 958131157, 958735681, 960269377, + 960946321, 962442001, 962489557, 962523169, 964412837, 965501857, 967266451, 967287751, 967790401, 968283247, + 968413217, 968751241, 969528337, 970586713, 971975071, 974113601, 974471243, 974774401, 975576281, 976396961, + 977483449, 979363153, 980056507, 980725201, 981484561, 983456377, 984133441, 984252001, 985052881, 985075681, + 987842101, 994133479, 995586373, 995650921, 997836841, 998489017, 998590601, 998596741, 998724481, 999828727, + 1002261781, 1003062061, 1005402133, 1005833971, 1006800829, 1008777001, 1008839999, 1009025263, 1009140161, 1011319501, + 1011333061, 1011570457, 1011909271, 1012438391, 1013833153, 1015339441, 1015626151, 1017748057, 1020515761, 1021281301, + 1022336611, 1024041853, 1024123501, 1024605121, 1025035129, 1026738161, 1027744453, 1028494429, 1034252929, 1034958601, + 1040234231, 1049584313, 1050102901, 1050535501, 1054999441, 1055009117, 1056121453, 1057426651, 1063212481, 1065508321, + 1065602281, 1066972301, 1069388497, 1070639389, 1070941987, 1071512749, 1071643249, 1072898711, 1073159281, 1073288581, + 1073484823, 1075100041, 1077133397, 1078467589, 1081798061, 1082472553, 1084241341, 1084444481, 1090858081, 1093150081, + 1093352833, 1093526353, 1094042321, 1097416321, 1098743563, 1100624857, 1101623381, 1101673501, 1102573501, 1102750013, + 1104194521, 1105038871, 1106529761, 1106580817, 1106595493, 1107138961, 1108135381, 1109304913, 1110582947, 1111205873, + 1111939201, 1112671603, 1114277221, 1116379301, 1117202557, 1117785881, 1117828001, 1117890019, 1119412321, 1120076281, + 1120981021, 1121176981, 1123406047, 1123625501, 1123727617, 1124396521, 1125038377, 1127040769, 1130933429, 1134367777, + 1138289041, 1138607233, 1139137057, 1140573601, 1142466151, 1147434289, 1148578201, 1150229761, 1151670001, 1153164097, + 1153440289, 1154343961, 1154691409, 1154987209, 1155939709, 1156761911, 1156993373, 1157839381, 1159421509, 1160844821, + 1163098249, 1163227759, 1164218641, 1165717129, 1166475601, 1166598217, 1168221121, 1168256953, 1168492417, 1173229201, + 1173545533, 1174300093, 1180970407, 1181566219, 1183338241, 1184554801, 1186325981, 1187235193, 1191153937, 1191216133, + 1192314817, 1192412033, 1192903531, 1193229577, 1193557093, 1195524181, 1196852273, 1198650961, 1198880261, 1200456577, + 1200778753, 1202142061, 1204205449, 1205606533, 1205772499, 1209998077, 1210393801, 1210562701, 1210653541, 1213619761, + 1217181061, 1217823517, 1217924159, 1219816261, 1219858921, 1220114377, 1221127013, 1222861271, 1223531677, 1223941657, + 1225128829, 1226230297, 1226855293, 1227220801, 1229491063, 1229751667, 1230446653, 1231362793, 1232445677, 1234125721, + 1234646533, 1235188597, 1235864033, 1236313501, 1236442421, 1238825569, 1242171349, 1242858317, 1249166881, 1249785941, + 1250656621, 1252236421, 1254277909, 1255665613, 1257102001, 1258903981, 1260332137, 1263293281, 1264145401, 1265477791, + 1266003461, 1266273793, 1266425101, 1267345081, 1269295201, 1269835201, 1270193401, 1270489621, 1270667353, 1272558739, + 1272866167, 1282447477, 1282568741, 1285636801, 1286298133, 1286298263, 1296613501, 1297443913, 1299072721, 1299784141, + 1299963601, 1301509249, 1301926081, 1302745481, 1306836001, 1307004641, 1307520469, 1307823661, 1308758533, 1308998741, + 1309723213, 1309983901, 1310329567, 1311255661, 1311616153, 1312332001, 1312573123, 1313396221, 1315858381, 1316169541, + 1318126321, 1318717531, 1319978701, 1319992181, 1320793813, 1321058213, 1323668917, 1325172421, 1325329297, 1328256247, + 1329174601, 1329431689, 1331973329, 1341010577, 1341926401, 1343575381, 1344597577, 1344975721, 1345514101, 1345523401, + 1347387361, 1348964401, 1350685001, 1351126261, 1352453257, 1353051517, 1356241321, 1356328121, 1357459183, 1362463807, + 1362515701, 1362742561, 1365662917, 1366587661, 1366608377, 1368769681, 1371908137, 1372681861, 1375322101, 1376799577, + 1378646179, 1379464633, 1382453333, 1383283129, 1385656829, 1386705433, 1388972353, 1389353941, 1389975149, 1391890033, + 1393851553, 1394640941, 1394746081, 1394942473, 1397357851, 1398883201, 1400859847, 1401840833, 1404008369, 1404253369, + 1406826241, 1406851249, 1409372779, 1413803197, 1414154827, 1414529533, 1415969101, 1417986901, 1421475031, 1424503849, + 1425860101, 1426319563, 1426534201, 1427771089, 1428966001, 1432354901, 1435091377, 1438648993, 1440231941, 1440922891, + 1441139641, 1441678411, 1442945689, 1443388481, 1443742273, 1446298309, 1446434677, 1446818651, 1448921633, 1451635201, + 1454282449, 1454445413, 1456527461, 1457378449, 1461307717, 1463065501, 1463178817, 1463992661, 1464568381, 1465908193, + 1465945417, 1468540477, 1468824787, 1469059481, 1469960377, 1470080501, 1470650851, 1471628401, 1472221921, 1473580001, + 1477289941, 1481626513, 1482274513, 1482876673, 1483873861, 1483918801, 1485061471, 1486564301, 1493114149, 1495190699, + 1497221281, 1497965713, 1499971457, 1499989177, 1500142001, 1501165097, 1502171117, 1502403121, 1503240559, 1503705601, + 1504139521, 1504832033, 1507746241, 1509156013, 1510870241, 1511558533, 1515175087, 1515785041, 1517039371, 1518014689, + 1518290707, 1520190341, 1521221473, 1522302121, 1526732803, 1529648231, 1529819971, 1530495289, 1532419099, 1532569681, + 1532755369, 1533343261, 1534063081, 1535020133, 1536112001, 1536251047, 1536883357, 1537433899, 1537641691, 1538012449, + 1539583921, 1539804001, 1540454761, 1540550413, 1541047813, 1541849761, 1541955409, 1544145121, 1545019813, 1545177581, + 1546106773, 1546340401, 1546508057, 1547140841, 1547543161, 1547712601, 1550924873, 1554270481, 1557118081, 1560312001, + 1560620041, 1561800833, 1565893201, 1566594551, 1567830241, 1568916311, 1574362441, 1574601601, 1577983489, 1578009401, + 1580449201, 1581576641, 1581714481, 1582783777, 1583230241, 1583658649, 1586436193, 1587650401, 1590394313, 1593706201, + 1595647351, 1595887921, 1598197201, 1602517949, 1603765021, 1603810561, 1603994701, 1609916491, 1609935913, 1612121473, + 1614508267, 1617795181, 1617921667, 1619447741, 1620646177, 1627103521, 1627898401, 1628692201, 1630062253, 1630307617, + 1631314609, 1632286673, 1632513601, 1633044241, 1636185601, 1637434657, 1637436457, 1637930893, 1638294661, 1639351981, + 1639846391, 1641971701, 1642814653, 1644637051, 1645413001, 1647225529, 1648076041, 1649430889, 1650265549, 1650682153, + 1654940509, 1655660761, 1656229921, 1656280033, 1656917377, 1659009601, 1661202113, 1668037621, 1668926629, 1669893661, + 1671603667, 1671714241, 1672125131, 1674091141, 1674658133, 1675978193, 1678274581, 1679130641, 1680901381, 1683174533, + 1685433413, 1686001861, 1687248001, 1691745821, 1692605041, 1694128129, 1695158921, 1696893101, 1698707377, 1699279441, + 1700250049, 1709909293, 1710753001, 1712392321, 1714322377, 1716160321, 1716714793, 1716774481, 1718013133, 1718088301, + 1719197621, 1721061497, 1721986313, 1722007169, 1722685777, 1725675451, 1726372441, 1731048937, 1731995497, 1732924001, + 1734059291, 1734285601, 1735071913, 1736481601, 1738687469, 1740214841, 1742288881, 1742815621, 1743166441, 1744605097, + 1746692641, 1746721681, 1749124829, 1750412161, 1754818561, 1757148121, 1760014561, 1766984389, 1767234613, 1769091241, + 1769267761, 1770236893, 1771303801, 1772267281, 1773582977, 1776439261, 1776820033, 1779649381, 1779892577, 1784306273, + 1784638309, 1785843547, 1786005521, 1787934881, 1790023861, 1791426787, 1792442737, 1792588813, 1794814103, 1801558201, + 1801774081, 1802510669, 1803768091, 1804906517, 1805947313, 1809888967, 1816408273, 1817067169, 1819829749, 1820306953, + 1821514633, 1828682101, 1828887061, 1831258601, 1835114401, 1837156049, 1837599769, 1839568981, 1841034961, 1841099261, + 1841479501, 1844028961, 1846171781, 1847811673, 1849964117, 1850233897, 1850598961, 1852496761, 1853926777, 1854084649, + 1854940231, 1856689453, 1857221281, 1858098497, 1858197961, 1860373241, 1861026133, 1861880689, 1862880401, 1866409861, + 1867906721, 1868682241, 1871987041, 1872937057, 1873177693, 1874634721, 1874849929, 1878691753, 1879111697, 1879623157, + 1879775501, 1883509633, 1883785681, 1885915841, 1894909141, 1894955311, 1897700113, 1899081757, 1899525601, 1900687381, + 1903447841, 1904658913, 1905958891, 1908088001, 1909566073, 1910134309, 1911197947, 1912950241, 1914303841, 1915391521, + 1916987593, 1917397637, 1920301951, 1921309633, 1922092567, 1922687293, 1923224689, 1923311317, 1923845801, 1924201501, + 1925042737, 1928903971, 1929862849, 1930403333, 1930447501, 1930534453, 1930915169, 1934350351, 1938264241, 1940048881, + 1943951041, 1944125633, 1945042181, 1950987193, 1952513369, 1952968753, 1957705177, 1959659857, 1960708261, 1963149553, + 1965007601, 1968002149, 1970065681, 1974474049, 1977257441, 1982123893, 1982826961, 1988071801, 1988713189, 1988835713, + 1988965861, 1989192277, 1991063449, 1995784961, 1995830761, 1996231189, 1996339649, 1997844157, 1998780001, 1999053601, + 1999111801, 1999743661, 2004299641, 2007646961, 2013554869, 2013834961, 2016481477, 2017021333, 2017509601, 2019564769, + 2021392369, 2021884343, 2027675701, 2028279793, 2028631361, 2028812399, 2029830409, 2030600833, 2036224321, 2043173273, + 2049293401, 2050617713, 2052149221, 2054711381, 2055634561, 2057267941, 2057835781, 2058072041, 2059739221, 2062612033, + 2068867841, 2070739441, 2072624761, 2076192007, 2081039297, 2081551753, 2082146617, 2083034113, 2083997441, 2085453649, + 2085882661, 2086645009, 2093300401, 2095627153, 2096046457, 2097317377, 2100292841, 2101470541, 2101744837, 2104994449, + 2106147457, 2107148761, 2114643217, 2115769633, 2115986557, 2116483027, 2116541221, 2117031263, 2117555641, 2118621097, + 2120096161, 2123601751, 2124078653, 2124691213, 2127197489, 2128104001, 2129304997, 2130134533, 2131004737, 2131811501, + 2140699681, 2140771609, 2141340833, 2144961253, 2147418113, 2147429509, 2152627801, 2154446641, 2155416251, 2156151313, + 2164282177, 2168431201, 2170282969, 2172155819, 2173499329, 2173540951, 2173579801, 2175126601, 2175406201, 2175646177, + 2177374321, 2177645557, 2178082901, 2178939221, 2180221201, 2182281601, 2182802689, 2185362233, 2187717761, 2193980881, + 2199617701, 2200115713, 2201924341, 2202101761, 2202205897, 2203649197, 2203856497, 2206095589, 2210578759, 2213431729, + 2216960929, 2217879901, 2219072017, 2224252801, 2229468697, 2231332357, 2233031701, 2240507821, 2241880033, 2241982009, + 2244932281, 2245519981, 2246762899, 2248354153, 2251732033, 2254314241, 2254757077, 2256197761, 2256748777, 2256751837, + 2262861901, 2269307587, 2274584089, 2283289681, 2284416181, 2289251669, 2289624793, 2290316377, 2290910257, 2291205461, + 2292068143, 2295209281, 2296995121, 2299190401, 2300628601, 2300795353, 2301745249, 2304120001, 2308966661, 2309241601, + 2309405617, 2311558021, 2311575001, 2315137261, 2320527613, 2323147201, 2324867399, 2329584217, 2330569541, 2331181621, + 2335341601, 2338157597, 2338728001, 2340460487, 2345907961, 2347597981, 2352371251, 2354453561, 2355230749, 2355320101, + 2355622721, 2355649921, 2355735089, 2358534361, 2360261989, 2370771181, 2370928337, 2371350101, 2372976563, 2374232977, + 2375415841, 2377166401, 2378309041, 2381782597, 2382678101, 2383164577, 2385574201, 2389072321, 2389544977, 2393708761, + 2394311233, 2398393661, 2404912501, 2411128441, 2412172153, 2412675721, 2413973071, 2422296241, 2423401681, 2425249601, + 2428648967, 2428870753, 2428986913, 2429407961, 2430697513, 2431136401, 2431144801, 2432761633, 2432860273, 2433791593, + 2434964321, 2434974433, 2435091221, 2436691321, 2437907779, 2438778413, 2442050353, 2442454561, 2443708961, 2444950561, + 2448039497, 2448374689, 2453473049, 2454285751, 2456536681, 2457846161, 2463713281, 2471205361, 2473120961, 2473189441, + 2473823353, 2474308069, 2474676949, 2476283239, 2477814193, 2478643907, 2480147521, 2480343553, 2482435981, 2482682131, + 2484408301, 2486017249, 2488420801, 2488591117, 2492480233, 2494660033, 2494984321, 2495834329, 2499327041, 2501012599, + 2501771329, 2502525637, 2504008609, 2506529257, 2506733189, 2507121037, 2508178843, 2513230891, 2516684801, 2519297089, + 2525070241, 2526566041, 2528291341, 2529410281, 2529827821, 2529854713, 2530351561, 2532630787, 2533465661, 2533797017, + 2535516173, 2537105761, 2539406281, 2539736257, 2540469901, 2541660367, 2542479481, 2544590161, 2545934077, 2548051801, + 2550139253, 2550780277, 2551365769, 2552418761, 2553272929, 2555391481, 2561945401, 2564536201, 2565186137, 2570087521, + 2571180247, 2575060949, 2575737361, 2577345541, 2582092189, 2582246701, 2582952769, 2583322381, 2584460701, 2588054401, + 2588582089, 2590663681, 2593065721, 2595276353, 2597289241, 2597294701, 2598933481, 2600611861, 2602343521, 2602378721, + 2604465013, 2604803701, 2611122229, 2611461529, 2613382201, 2614688801, 2616180821, 2617563031, 2621080741, 2621977627, + 2622993661, 2624549929, 2625903601, 2626783921, 2627284987, 2630643401, 2632605049, 2634284801, 2634804481, 2634820813, + 2638067881, 2639099233, 2642159809, 2642582251, 2646751249, 2646790033, 2648662777, 2649907201, 2650820329, 2651507713, + 2654716321, 2656494271, 2658630913, 2658696301, 2659265701, 2668095181, 2668469431, 2670972949, 2672605657, 2672651521, + 2676053333, 2677147201, 2677821121, 2678785621, 2681041843, 2682823681, 2683742491, 2684284441, 2687655169, 2688124001, + 2689427281, 2690408533, 2690867401, 2693739751, 2695115473, 2700818017, 2700891839, 2701878941, 2704957909, 2706863833, + 2707661501, 2716157989, 2716275007, 2717428033, 2719319513, 2721666817, 2721721939, 2723859001, 2725357249, 2733156029, + 2736316301, 2738184697, 2740336561, 2744329909, 2746021741, 2753333227, 2753538001, 2759392633, 2765323397, 2766006253, + 2767672189, 2769080161, 2769602333, 2774295577, 2777887297, 2778304273, 2779477741, 2781117721, 2781226477, 2786028337, + 2787998641, 2789218909, 2800352011, 2805762961, 2809635901, 2812672981, 2814748201, 2823570433, 2824256377, 2824804693, + 2824854913, 2828205397, 2832384133, 2832743713, 2837697773, 2837917633, 2840634109, 2840871041, 2841190381, 2847894377, + 2848466281, 2848722131, 2855046421, 2855071801, 2855512909, 2862066481, 2865483601, 2866005139, 2866527841, 2870377309, + 2871536561, 2872527733, 2872948321, 2874382853, 2877769501, 2881429741, 2882370481, 2885594497, 2887955533, 2890316801, + 2890414873, 2892426029, 2894667781, 2895004927, 2899294889, 2903776129, 2915953633, 2916247819, 2918295451, 2920691161, + 2923042141, 2924158001, 2929062533, 2929106753, 2930831641, 2931708097, 2932327549, 2936227603, 2936958181, 2941174897, + 2941343633, 2944555681, 2944677961, 2945208001, 2945549881, 2951136343, 2956724317, 2957320351, 2965700233, 2967053953, + 2968206601, 2974506841, 2975377429, 2976930001, 2978766341, 2980689601, 2986025677, 2987414977, 2990152901, 2993462713, + 2993495041, 2994098281, 2994415201, 2998202353, 2998919873, 3000688381, 3001561441, 3002647829, 3004443679, 3009628301, + 3011421841, 3014101261, 3015502181, 3016957381, 3017444761, 3018147217, 3018576689, 3019916461, 3025350343, 3026575553, + 3028586471, 3030393901, 3033332641, 3034402681, 3034817209, 3035375047, 3036079729, 3037295801, 3037781251, 3038880473, + 3039681457, 3041984353, 3042630533, 3048159841, 3050190163, 3056100623, 3056160929, 3057886591, 3058670677, 3059397793, + 3063685633, 3065998717, 3076505209, 3077122133, 3079496551, 3082054697, 3082068013, 3083053387, 3083537689, 3083884651, + 3088408429, 3089013313, 3091019777, 3094763851, 3099670657, 3103800701, 3112974481, 3114125071, 3115667521, 3120445697, + 3122287981, 3129914881, 3133899409, 3135040133, 3143282221, 3145410761, 3150972917, 3156599161, 3156643141, 3157579861, + 3163106953, 3166504273, 3167442721, 3170262409, 3172658653, 3175204531, 3175255717, 3178375201, 3181356263, 3181391641, + 3182606857, 3182655361, 3182891401, 3185472001, 3187035113, 3187421077, 3187939921, 3196397821, 3196431829, 3197565001, + 3197632441, 3197911001, 3197911741, 3199164901, 3205663921, 3207297773, 3208902491, 3212465437, 3215031751, 3217412881, + 3219808411, 3221580281, 3222693421, 3224143441, 3225081473, 3227082823, 3227209057, 3229131137, 3233558021, 3237992101, + 3242533897, 3248236309, 3250348417, 3250700737, 3252148621, 3257334541, 3258647809, 3258892801, 3261114601, 3263097641, + 3263568901, 3263626957, 3264820001, 3265122451, 3267417677, 3268506541, 3268841941, 3270933121, 3271999249, 3272030401, + 3272702497, 3274264033, 3275671969, 3276075709, 3277047649, 3278640289, 3280067129, 3282974857, 3287174129, 3288757249, + 3295362727, 3296403601, 3299246833, 3302322241, 3304307341, 3305829073, 3306686659, 3306957593, 3310858777, 3312489577, + 3312536569, 3313196881, 3315139717, 3320669437, 3323308501, 3323590463, 3323829169, 3328354801, 3332800021, 3334350781, + 3340214413, 3342005633, 3344191241, 3346172189, 3347908801, 3349218881, 3350993969, 3352091557, 3355382857, 3355953001, + 3357417181, 3359737921, 3360511981, 3369139201, 3371024521, 3371452921, 3371693063, 3372667121, 3373086601, 3381052177, + 3381901921, 3385842877, 3386603221, 3387014401, 3387487351, 3389030261, 3395091311, 3399205591, 3399890413, 3402234749, + 3407609221, 3407772817, 3407952169, 3408135121, 3409339393, 3411250081, 3411574801, 3411829693, 3412575097, 3415379701, + 3415832137, 3417522841, 3420143941, 3421845001, 3423222757, 3423580481, 3427050673, 3428133103, 3429457921, 3429982081, + 3430804297, 3432695921, 3432997537, 3433458073, 3434575327, 3435973837, 3440195713, 3443704261, 3449768513, 3450717901, + 3453900913, 3458257741, 3461861761, 3463907761, 3464236901, 3466158361, 3470716657, 3474335437, 3480174001, 3482161261, + 3485747521, 3489958697, 3491763493, 3492178873, 3492883081, 3493262761, 3497607433, 3504132113, 3512030497, 3512291021, + 3512369857, 3513604657, 3516565057, 3519318721, 3524086333, 3525088961, 3529119361, 3529864391, 3532687201, 3533662129, + 3533856913, 3538213381, 3542303047, 3543203333, 3548378341, 3549286001, 3549988261, 3552158521, 3553567057, 3557646401, + 3562963973, 3563340457, 3566428301, 3574891757, 3582711841, 3583249921, 3583604161, 3584800801, 3586833253, 3587553971, + 3589937261, 3590409439, 3593276353, 3594110081, 3596491907, 3596815169, 3598772761, 3602006101, 3605151241, 3611571121, + 3612298321, 3612825221, 3614770573, 3616574081, 3620631169, 3628526287, 3630596257, 3631828481, 3632452741, 3635993089, + 3649116277, 3649965281, 3650158849, 3651572609, 3656355841, 3658730893, 3662387977, 3662503093, 3663084541, 3668926801, + 3669587533, 3672754633, 3677180797, 3679657997, 3682471321, 3685647701, 3685775741, 3692307161, 3695628133, 3697278427, + 3700801861, 3705582073, 3705623281, 3708123301, 3708905341, 3709626961, 3712887289, 3713287801, 3713448769, 3718226401, + 3721486081, 3723410161, 3723699373, 3725016749, 3727828501, 3729097633, 3733761739, 3736293461, 3745192001, 3746101189, + 3749383681, 3751554581, 3751782737, 3754680403, 3756668401, 3759781369, 3760622689, 3760896133, 3762110881, 3767640601, + 3773061337, 3774337201, 3784123501, 3787491457, 3798040471, 3798626833, 3799111681, 3800084401, 3805699501, 3807112123, + 3807308269, 3807749821, 3809018947, 3813919453, 3817561777, 3817706621, 3821233121, 3827035237, 3832807681, 3833208961, + 3842941741, 3846174151, 3846532801, 3847106803, 3850058689, 3852800033, 3863326897, 3865604023, 3867183937, 3874471147, + 3874523017, 3875096893, 3875965417, 3886515361, 3886643801, 3887423437, 3887635753, 3891892421, 3891919417, 3894053311, + 3896079281, 3897241129, 3897869201, 3898906129, 3900327241, 3903711841, 3905533721, 3905876501, 3907577521, 3907752241, + 3912174421, 3914880337, 3914923211, 3915467341, 3915604421, 3915921241, 3918227437, 3922321561, 3926912669, 3929293061, + 3934940833, 3935864017, 3936123601, 3945165841, 3947233201, 3947383201, 3953408801, 3953949421, 3955572001, 3958597301, + 3958930441, 3959578801, 3960728641, 3962037061, 3966350203, 3967343161, 3971095301, 3973556837, 3979485931, 3982017601, + 3987528793, 3987960913, 3991124341, 3992697997, 3997536427, 4005660961, 4007365741, 4011996871, 4015548769, 4017684529, + 4018283501, 4020144133, 4026822577, 4027012021, 4027518961, 4028465873, 4028771849, 4031223841, 4034969401, 4034993269, + 4035498409, 4036395581, 4042538497, 4044601751, 4044884689, 4048493983, 4053267217, 4054039841, 4057195309, 4058433931, + 4059776533, 4060942381, 4061009971, 4064633821, 4067039461, 4067887501, 4068671881, 4071644893, 4075241633, 4075721921, + 4076009857, 4079665633, 4079682361, 4083376067, 4085074909, 4088147617, 4088838913, 4092929149, 4098258707, 4099180801, + 4100934241, 4103745689, 4105691393, 4108970251, 4109461709, 4109711581, 4110320663, 4113013141, 4115891893, 4117058221, + 4117447441, 4121286907, 4127050621, 4129914673, 4133928761, 4135847101, 4136916001, 4137262541, 4138838401, 4139015987, + 4150174393, 4155375349, 4157008813, 4162880401, 4166032873, 4183664101, 4185636781, 4186561633, 4187360341, 4191864013, + 4192060699, 4195843037, 4196323561, 4204344601, 4206006229, 4206295433, 4212105409, 4215885697, 4218900001, 4220122321, + 4232966251, 4234224601, 4237212061, 4243744201, 4244022301, 4244663651, 4247990917, 4250920459, 4251904273, 4255695013, + 4257003353, 4261352869, 4271267333, 4275011401, 4277526901, 4278305651, 4282867213, 4285148981, 4293088801, 4294901761, + } + + primes16 = [65536]byte{ + 2, 1, 1, 2, 1, 2, 1, 4, 3, 2, // 0-9 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 10-19 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 20-29 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 30-39 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 40-49 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 50-59 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 60-69 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 70-79 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 80-89 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 90-99 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 100-109 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 110-119 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 120-129 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 130-139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 140-149 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 150-159 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 160-169 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 170-179 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 180-189 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 190-199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 200-209 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 210-219 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 220-229 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 230-239 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 240-249 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 250-259 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 260-269 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 270-279 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 280-289 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 290-299 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 300-309 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 310-319 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 320-329 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 330-339 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 340-349 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 350-359 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 360-369 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 370-379 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 380-389 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 390-399 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 400-409 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 410-419 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 420-429 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 430-439 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 440-449 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 450-459 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 460-469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 470-479 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 480-489 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 490-499 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 500-509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 510-519 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 520-529 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 530-539 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 540-549 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 550-559 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 560-569 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 570-579 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 580-589 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 590-599 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 600-609 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 610-619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 620-629 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 630-639 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 640-649 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 650-659 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 660-669 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 670-679 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 680-689 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 690-699 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 700-709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 710-719 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 720-729 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 730-739 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 740-749 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 750-759 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 760-769 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 770-779 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 780-789 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 790-799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 800-809 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 810-819 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 820-829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 830-839 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 840-849 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 850-859 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 860-869 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 870-879 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 880-889 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 890-899 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 900-909 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 910-919 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 920-929 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 930-939 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 940-949 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 950-959 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 960-969 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 970-979 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 980-989 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 990-999 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 1000-1009 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 1010-1019 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1020-1029 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 1030-1039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 1040-1049 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1050-1059 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 1060-1069 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 1070-1079 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 1080-1089 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 1090-1099 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 1100-1109 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1110-1119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 1120-1129 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 1130-1139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1140-1149 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 1150-1159 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 1160-1169 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1170-1179 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1180-1189 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 1190-1199 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 1200-1209 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 1210-1219 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 1220-1229 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 1230-1239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 1240-1249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 1250-1259 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 1260-1269 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 1270-1279 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 1280-1289 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 1290-1299 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 1300-1309 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 1310-1319 + 1, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 1320-1329 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 1330-1339 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 1340-1349 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1350-1359 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1360-1369 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 1370-1379 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 1380-1389 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 1390-1399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 1400-1409 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 1410-1419 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 1420-1429 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 1430-1439 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 1440-1449 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 1450-1459 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1460-1469 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1470-1479 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 1480-1489 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 1490-1499 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1500-1509 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 1510-1519 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 1520-1529 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 1530-1539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 1540-1549 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 1550-1559 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 1560-1569 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 1570-1579 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 1580-1589 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 1590-1599 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 1600-1609 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 1610-1619 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 1620-1629 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 1630-1639 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 1640-1649 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1650-1659 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 24, // 1660-1669 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 1670-1679 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 1680-1689 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 1690-1699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 1700-1709 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1710-1719 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 1720-1729 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 1730-1739 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1740-1749 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 1750-1759 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 1760-1769 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1770-1779 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 1780-1789 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1790-1799 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1800-1809 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 1810-1819 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 1820-1829 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 1830-1839 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 1840-1849 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1850-1859 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 1860-1869 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 1870-1879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 1880-1889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1890-1899 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1900-1909 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 1910-1919 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 1920-1929 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 1930-1939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 1940-1949 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 1950-1959 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 1960-1969 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 1970-1979 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 1980-1989 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 1990-1999 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 2000-2009 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 2010-2019 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 2020-2029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 2030-2039 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 2040-2049 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 2050-2059 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 2060-2069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2070-2079 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 2080-2089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 2090-2099 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2100-2109 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 2110-2119 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 2120-2129 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2130-2139 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 2140-2149 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 2150-2159 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 2160-2169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 2170-2179 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 2180-2189 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 2190-2199 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 2200-2209 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 2210-2219 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 2220-2229 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 2230-2239 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 2240-2249 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 2250-2259 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 2260-2269 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 2270-2279 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 2280-2289 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 2290-2299 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 2300-2309 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 2310-2319 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 2320-2329 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 2330-2339 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2340-2349 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 2350-2359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2360-2369 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2370-2379 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 2380-2389 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 2390-2399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2400-2409 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 2410-2419 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 2420-2429 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2430-2439 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 2440-2449 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 2450-2459 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 2460-2469 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 2470-2479 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 2480-2489 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 2490-2499 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 2500-2509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2510-2519 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2520-2529 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 2530-2539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 2540-2549 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 2550-2559 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 2560-2569 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 2570-2579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2580-2589 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 2590-2599 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 2600-2609 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2610-2619 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 2620-2629 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 2630-2639 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 2640-2649 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 2650-2659 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 2660-2669 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 2670-2679 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 2680-2689 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 2690-2699 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2700-2709 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 2710-2719 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 2720-2729 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 2730-2739 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 2740-2749 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 2750-2759 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 2760-2769 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 2770-2779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 2780-2789 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2790-2799 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 2800-2809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 2810-2819 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 2820-2829 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 2830-2839 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 2840-2849 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 2850-2859 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 2860-2869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 2870-2879 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 2880-2889 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 2890-2899 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 2900-2909 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 2910-2919 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 2920-2929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 2930-2939 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 2940-2949 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 2950-2959 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 2960-2969 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 2970-2979 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 2980-2989 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 2990-2999 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3000-3009 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 3010-3019 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 3020-3029 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3030-3039 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 3040-3049 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3050-3059 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 3060-3069 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 3070-3079 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 3080-3089 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 3090-3099 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 3100-3109 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 3110-3119 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 3120-3129 + 7, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 3130-3139 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 3140-3149 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 3150-3159 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 3160-3169 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3170-3179 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3180-3189 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 3190-3199 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 3200-3209 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3210-3219 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 3220-3229 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 3230-3239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3240-3249 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 3250-3259 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3260-3269 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 3270-3279 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 3280-3289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 3290-3299 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 3300-3309 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 3310-3319 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 3320-3329 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 3330-3339 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 3340-3349 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 3350-3359 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3360-3369 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 3370-3379 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 3380-3389 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 3390-3399 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 3400-3409 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 3410-3419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 3420-3429 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 3430-3439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 3440-3449 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3450-3459 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 3460-3469 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 3470-3479 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3480-3489 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 3490-3499 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3500-3509 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 3510-3519 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 3520-3529 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 3530-3539 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 3540-3549 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 3550-3559 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3560-3569 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3570-3579 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 3580-3589 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 3590-3599 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 3600-3609 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 3610-3619 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 3620-3629 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 3630-3639 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 3640-3649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 3650-3659 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3660-3669 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 3670-3679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3680-3689 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3690-3699 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 3700-3709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 3710-3719 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 3720-3729 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 3730-3739 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 3740-3749 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3750-3759 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 3760-3769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 3770-3779 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 3780-3789 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 3790-3799 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 3800-3809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3810-3819 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 3820-3829 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 3830-3839 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3840-3849 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 3850-3859 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 3860-3869 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3870-3879 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 3880-3889 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 3890-3899 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 3900-3909 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 3910-3919 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 3920-3929 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 3930-3939 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 3940-3949 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 3950-3959 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 3960-3969 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 3970-3979 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 3980-3989 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 3990-3999 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 4000-4009 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 4010-4019 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 4020-4029 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 4030-4039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 4040-4049 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 4050-4059 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 4060-4069 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 4070-4079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4080-4089 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 4090-4099 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4100-4109 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 4110-4119 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 4120-4129 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 4130-4139 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 4140-4149 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 18, // 4150-4159 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 4160-4169 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 4170-4179 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 4180-4189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4190-4199 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4200-4209 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 4210-4219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 4220-4229 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4230-4239 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 4240-4249 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 4250-4259 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4260-4269 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 4270-4279 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 4280-4289 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 4290-4299 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 4300-4309 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 4310-4319 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 4320-4329 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 4330-4339 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 4340-4349 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 4350-4359 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 4360-4369 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 4370-4379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4380-4389 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 4390-4399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 4400-4409 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4410-4419 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 4420-4429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4430-4439 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 4440-4449 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 4450-4459 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 4460-4469 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4470-4479 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 4480-4489 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 4490-4499 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 4500-4509 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 4510-4519 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 4520-4529 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 4530-4539 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 4540-4549 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4550-4559 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 4560-4569 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 4570-4579 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 4580-4589 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 4590-4599 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 4600-4609 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4610-4619 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 4620-4629 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 4630-4639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 4640-4649 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 4650-4659 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 4660-4669 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 4670-4679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4680-4689 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 4690-4699 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 4700-4709 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4710-4719 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 4720-4729 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 4730-4739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4740-4749 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 4750-4759 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 4760-4769 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 4770-4779 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 4780-4789 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 4790-4799 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 4800-4809 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 4810-4819 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4820-4829 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 4830-4839 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 4840-4849 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4850-4859 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4860-4869 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 4870-4879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 4880-4889 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 4890-4899 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 4900-4909 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 4910-4919 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 4920-4929 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 4930-4939 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 4940-4949 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 4950-4959 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 4960-4969 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 4970-4979 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 4980-4989 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 4990-4999 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 5000-5009 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5010-5019 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 5020-5029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 5030-5039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5040-5049 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 5050-5059 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5060-5069 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5070-5079 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 5080-5089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 5090-5099 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5100-5109 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 28, // 5110-5119 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 5120-5129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5130-5139 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5140-5149 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 5150-5159 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5160-5169 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 5170-5179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 5180-5189 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 5190-5199 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 5200-5209 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5210-5219 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5220-5229 + 1, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 5230-5239 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 5240-5249 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5250-5259 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 5260-5269 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 5270-5279 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5280-5289 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5290-5299 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 5300-5309 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 5310-5319 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 5320-5329 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 5330-5339 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5340-5349 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 5350-5359 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 5360-5369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5370-5379 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5380-5389 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 5390-5399 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5400-5409 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 5410-5419 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5420-5429 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5430-5439 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 5440-5449 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 5450-5459 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5460-5469 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 5470-5479 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 5480-5489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5490-5499 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 5500-5509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 5510-5519 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5520-5529 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 5530-5539 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5540-5549 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5550-5559 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 5560-5569 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 5570-5579 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5580-5589 + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 5590-5599 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 5600-5609 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 5610-5619 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 5620-5629 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 5630-5639 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5640-5649 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 5650-5659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 5660-5669 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 5670-5679 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 5680-5689 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 5690-5699 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5700-5709 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 5710-5719 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5720-5729 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5730-5739 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 30, // 5740-5749 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 5750-5759 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 5760-5769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 5770-5779 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 5780-5789 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5790-5799 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5800-5809 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 5810-5819 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 5820-5829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 5830-5839 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 5840-5849 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 5850-5859 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 5860-5869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 5870-5879 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5880-5889 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 5890-5899 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 5900-5909 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 5910-5919 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 5920-5929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 5930-5939 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 5940-5949 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 5950-5959 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 5960-5969 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 5970-5979 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 5980-5989 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 5990-5999 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 6000-6009 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 6010-6019 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 6020-6029 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6030-6039 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 6040-6049 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 6050-6059 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6060-6069 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 6070-6079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 6080-6089 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6090-6099 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 6100-6109 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 6110-6119 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6120-6129 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 6130-6139 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 6140-6149 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 6150-6159 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 6160-6169 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 6170-6179 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 6180-6189 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 6190-6199 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 6200-6209 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 6210-6219 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 6220-6229 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 6230-6239 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 6240-6249 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6250-6259 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 6260-6269 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 6270-6279 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 6280-6289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 6290-6299 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6300-6309 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6310-6319 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 6320-6329 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6330-6339 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 6340-6349 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 6350-6359 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6360-6369 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 6370-6379 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 6380-6389 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 6390-6399 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 6400-6409 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6410-6419 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 6420-6429 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 6430-6439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 6440-6449 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 6450-6459 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 6460-6469 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 6470-6479 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6480-6489 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 6490-6499 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 6500-6509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6510-6519 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 6520-6529 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 6530-6539 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 6540-6549 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 6550-6559 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 6560-6569 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 6570-6579 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 6580-6589 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 6590-6599 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 6600-6609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 6610-6619 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 6620-6629 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 6630-6639 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 6640-6649 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 6650-6659 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 6660-6669 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 6670-6679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 6680-6689 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6690-6699 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 6700-6709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 6710-6719 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 6720-6729 + 3, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 6730-6739 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 6740-6749 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6750-6759 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 6760-6769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 6770-6779 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 6780-6789 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 6790-6799 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 6800-6809 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 6810-6819 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 6820-6829 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 6830-6839 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 6840-6849 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6850-6859 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 6860-6869 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 6870-6879 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 6880-6889 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 6890-6899 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 6900-6909 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 6910-6919 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 6920-6929 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 6930-6939 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 6940-6949 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 6950-6959 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 6960-6969 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 6970-6979 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 6980-6989 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 6990-6999 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7000-7009 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 7010-7019 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 7020-7029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 7030-7039 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 7040-7049 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 7050-7059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 7060-7069 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 7070-7079 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 7080-7089 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7090-7099 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 7100-7109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7110-7119 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 7120-7129 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 7130-7139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7140-7149 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 7150-7159 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 7160-7169 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 7170-7179 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 7180-7189 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 7190-7199 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 7200-7209 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 7210-7219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 7220-7229 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 7230-7239 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 7240-7249 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 7250-7259 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 7260-7269 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7270-7279 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 7280-7289 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 7290-7299 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 7300-7309 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7310-7319 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7320-7329 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 7330-7339 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 7340-7349 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 7350-7359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 7360-7369 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 7370-7379 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7380-7389 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 7390-7399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7400-7409 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 7410-7419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7420-7429 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 7430-7439 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7440-7449 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 7450-7459 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 7460-7469 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 7470-7479 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 7480-7489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 7490-7499 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 7500-7509 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 7510-7519 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 7520-7529 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 7530-7539 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 7540-7549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 7550-7559 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7560-7569 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 7570-7579 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 7580-7589 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7590-7599 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 7600-7609 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7610-7619 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 7620-7629 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 7630-7639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 7640-7649 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 7650-7659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 7660-7669 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 7670-7679 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 7680-7689 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 7690-7699 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 7700-7709 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 7710-7719 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 7720-7729 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7730-7739 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7740-7749 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 30, // 7750-7759 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 7760-7769 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 7770-7779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 7780-7789 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 7790-7799 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 7800-7809 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 7810-7819 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 7820-7829 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7830-7839 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7840-7849 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 7850-7859 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 7860-7869 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 7870-7879 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 7880-7889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 7890-7899 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 7900-7909 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 7910-7919 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 7920-7929 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 7930-7939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 7940-7949 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7950-7959 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 7960-7969 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 7970-7979 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 7980-7989 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 7990-7999 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 8000-8009 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 8010-8019 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 8020-8029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 8030-8039 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8040-8049 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 8050-8059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 8060-8069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8070-8079 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 8080-8089 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 8090-8099 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8100-8109 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 8110-8119 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 8120-8129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 8130-8139 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 8140-8149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8150-8159 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 8160-8169 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 8170-8179 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8180-8189 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 8190-8199 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 8200-8209 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 8210-8219 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8220-8229 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 8230-8239 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 8240-8249 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8250-8259 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 8260-8269 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 8270-8279 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 8280-8289 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 8290-8299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8300-8309 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 8310-8319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 8320-8329 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 8330-8339 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8340-8349 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 8350-8359 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 8360-8369 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 8370-8379 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 30, // 8380-8389 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 8390-8399 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 8400-8409 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 8410-8419 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 8420-8429 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8430-8439 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 8440-8449 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8450-8459 + 1, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 8460-8469 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 8470-8479 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 8480-8489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8490-8499 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8500-8509 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 8510-8519 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 8520-8529 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 8530-8539 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 8540-8549 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8550-8559 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 8560-8569 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 8570-8579 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 8580-8589 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 8590-8599 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 8600-8609 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8610-8619 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 8620-8629 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8630-8639 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 8640-8649 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8650-8659 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 8660-8669 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 8670-8679 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 8680-8689 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 8690-8699 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 8700-8709 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 8710-8719 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8720-8729 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 8730-8739 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 8740-8749 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 8750-8759 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 8760-8769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 8770-8779 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 8780-8789 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8790-8799 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 8800-8809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 8810-8819 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8820-8829 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 8830-8839 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 8840-8849 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8850-8859 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 8860-8869 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 8870-8879 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 8880-8889 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 8890-8899 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 8900-8909 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8910-8919 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 8920-8929 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 8930-8939 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 8940-8949 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 8950-8959 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 8960-8969 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 8970-8979 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 8980-8989 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 8990-8999 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 9000-9009 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 9010-9019 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 9020-9029 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9030-9039 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 9040-9049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 9050-9059 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 9060-9069 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 9070-9079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9080-9089 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9090-9099 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 9100-9109 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9110-9119 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 9120-9129 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 9130-9139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9140-9149 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 9150-9159 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9160-9169 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 9170-9179 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 9180-9189 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 9190-9199 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 9200-9209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9210-9219 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 9220-9229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 9230-9239 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9240-9249 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 9250-9259 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9260-9269 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 9270-9279 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 9280-9289 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 9290-9299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9300-9309 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 9310-9319 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 9320-9329 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 9330-9339 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 9340-9349 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 9350-9359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9360-9369 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 9370-9379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9380-9389 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 9390-9399 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 9400-9409 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 9410-9419 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9420-9429 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 9430-9439 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 9440-9449 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9450-9459 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 9460-9469 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 9470-9479 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9480-9489 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 9490-9499 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9500-9509 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9510-9519 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9520-9529 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 9530-9539 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 9540-9549 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 9550-9559 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 9560-9569 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9570-9579 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 9580-9589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9590-9599 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9600-9609 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 9610-9619 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 9620-9629 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9630-9639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 9640-9649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9650-9659 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9660-9669 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 9670-9679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 9680-9689 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 9690-9699 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 9700-9709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 9710-9719 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9720-9729 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 9730-9739 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 9740-9749 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9750-9759 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 9760-9769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9770-9779 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 9780-9789 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9790-9799 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 9800-9809 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 9810-9819 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 9820-9829 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 9830-9839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9840-9849 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 9850-9859 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9860-9869 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9870-9879 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 9880-9889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9890-9899 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 9900-9909 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 9910-9919 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 9920-9929 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 9930-9939 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 9940-9949 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9950-9959 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 9960-9969 + 3, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 9970-9979 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 9980-9989 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 9990-9999 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 28, // 10000-10009 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 10010-10019 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 10020-10029 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 10030-10039 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 10040-10049 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10050-10059 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 10060-10069 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 10070-10079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10080-10089 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 10090-10099 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 10100-10109 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 10110-10119 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10120-10129 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 10130-10139 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10140-10149 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 10150-10159 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 10160-10169 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 10170-10179 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10180-10189 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 10190-10199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10200-10209 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10210-10219 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 10220-10229 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10230-10239 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 10240-10249 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 10250-10259 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 10260-10269 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 10270-10279 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 10280-10289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10290-10299 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 10300-10309 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 10310-10319 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10320-10329 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 10330-10339 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 10340-10349 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 10350-10359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 10360-10369 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 10370-10379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10380-10389 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 28, // 10390-10399 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 10400-10409 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 10410-10419 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 10420-10429 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 10430-10439 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10440-10449 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 10450-10459 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 10460-10469 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 10470-10479 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 10480-10489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 10490-10499 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10500-10509 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 10510-10519 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 10520-10529 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 10530-10539 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 10540-10549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 10550-10559 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 10560-10569 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 10570-10579 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 10580-10589 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 10590-10599 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 10600-10609 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 10610-10619 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 10620-10629 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 10630-10639 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10640-10649 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 10650-10659 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 10660-10669 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 10670-10679 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 10680-10689 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 10690-10699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 10700-10709 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10710-10719 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 10720-10729 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 10730-10739 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10740-10749 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 10750-10759 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10760-10769 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10770-10779 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 10780-10789 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 32, // 10790-10799 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 10800-10809 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 10810-10819 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 10820-10829 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 10830-10839 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 10840-10849 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 10850-10859 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 10860-10869 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10870-10879 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 10880-10889 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10890-10899 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 28, // 10900-10909 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 10910-10919 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 10920-10929 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 10930-10939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 10940-10949 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 10950-10959 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 10960-10969 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 10970-10979 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 10980-10989 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 10990-10999 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 11000-11009 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11010-11019 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 11020-11029 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11030-11039 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 11040-11049 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 11050-11059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 11060-11069 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11070-11079 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 11080-11089 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 11090-11099 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11100-11109 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 11110-11119 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11120-11129 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 11130-11139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 11140-11149 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 11150-11159 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11160-11169 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 11170-11179 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11180-11189 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 11190-11199 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11200-11209 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 11210-11219 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 11220-11229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 11230-11239 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 11240-11249 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 11250-11259 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11260-11269 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 11270-11279 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 11280-11289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 11290-11299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11300-11309 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 11310-11319 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 11320-11329 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 11330-11339 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11340-11349 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 11350-11359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 11360-11369 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11370-11379 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 11380-11389 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 11390-11399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11400-11409 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11410-11419 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 11420-11429 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 11430-11439 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 11440-11449 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11450-11459 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 11460-11469 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11470-11479 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 11480-11489 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 11490-11499 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 11500-11509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 11510-11519 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 11520-11529 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 11530-11539 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 11540-11549 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 11550-11559 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 11560-11569 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 11570-11579 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 11580-11589 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 11590-11599 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11600-11609 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 11610-11619 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11620-11629 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 11630-11639 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11640-11649 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 11650-11659 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11660-11669 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 11670-11679 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 11680-11689 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 11690-11699 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11700-11709 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 11710-11719 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11720-11729 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11730-11739 + 3, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 11740-11749 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 11750-11759 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11760-11769 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 11770-11779 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 11780-11789 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11790-11799 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 11800-11809 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 11810-11819 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 11820-11829 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 24, // 11830-11839 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 11840-11849 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11850-11859 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 11860-11869 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11870-11879 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 11880-11889 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 11890-11899 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 11900-11909 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11910-11919 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 11920-11929 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 11930-11939 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 11940-11949 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 11950-11959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 11960-11969 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 11970-11979 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 11980-11989 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 11990-11999 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12000-12009 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 12010-12019 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 12020-12029 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12030-12039 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 12040-12049 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 12050-12059 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12060-12069 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 12070-12079 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 12080-12089 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12090-12099 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 12100-12109 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 24, // 12110-12119 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 12120-12129 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12130-12139 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 12140-12149 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12150-12159 + 1, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 12160-12169 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 12170-12179 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 12180-12189 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12190-12199 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 12200-12209 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 12210-12219 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 12220-12229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 12230-12239 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12240-12249 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 12250-12259 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 12260-12269 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12270-12279 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 12280-12289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12290-12299 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 12300-12309 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12310-12319 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 12320-12329 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12330-12339 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 12340-12349 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 12350-12359 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12360-12369 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 12370-12379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12380-12389 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12390-12399 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 12400-12409 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 12410-12419 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12420-12429 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 12430-12439 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12440-12449 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 12450-12459 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12460-12469 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 12470-12479 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12480-12489 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12490-12499 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 12500-12509 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 12510-12519 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 12520-12529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 12530-12539 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12540-12549 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 12550-12559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 12560-12569 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12570-12579 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 12580-12589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12590-12599 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12600-12609 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 12610-12619 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 12620-12629 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12630-12639 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12640-12649 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 12650-12659 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12660-12669 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 12670-12679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 12680-12689 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12690-12699 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 12700-12709 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 12710-12719 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 12720-12729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 12730-12739 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 12740-12749 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12750-12759 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 12760-12769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12770-12779 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12780-12789 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 12790-12799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 12800-12809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12810-12819 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 12820-12829 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12830-12839 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12840-12849 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 12850-12859 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 12860-12869 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 12870-12879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 12880-12889 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 12890-12899 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 12900-12909 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 12910-12919 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 12920-12929 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12930-12939 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 12940-12949 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 12950-12959 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 12960-12969 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 12970-12979 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 12980-12989 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 12990-12999 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 24, // 13000-13009 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 13010-13019 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13020-13029 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 13030-13039 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 13040-13049 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13050-13059 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 13060-13069 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 13070-13079 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13080-13089 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 13090-13099 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 13100-13109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13110-13119 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 13120-13129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 13130-13139 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 13140-13149 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 13150-13159 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 13160-13169 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 13170-13179 + 3, 2, 1, 4, 3, 2, 1, 30, 29, 28, // 13180-13189 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 13190-13199 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 13200-13209 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 13210-13219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 13220-13229 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13230-13239 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 13240-13249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 13250-13259 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 13260-13269 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 13270-13279 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13280-13289 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 13290-13299 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 13300-13309 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 13310-13319 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 13320-13329 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 28, // 13330-13339 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 13340-13349 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 13350-13359 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 13360-13369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13370-13379 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 13380-13389 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 13390-13399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13400-13409 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 13410-13419 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 13420-13429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13430-13439 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13440-13449 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 13450-13459 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 13460-13469 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 13470-13479 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 13480-13489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 13490-13499 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13500-13509 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 13510-13519 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 13520-13529 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 13530-13539 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13540-13549 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 13550-13559 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 13560-13569 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 13570-13579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13580-13589 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 13590-13599 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13600-13609 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 13610-13619 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 13620-13629 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 13630-13639 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 13640-13649 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 13650-13659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 13660-13669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 13670-13679 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 13680-13689 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 13690-13699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 13700-13709 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13710-13719 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 13720-13729 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 13730-13739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13740-13749 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 13750-13759 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 13760-13769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13770-13779 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 13780-13789 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 13790-13799 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 13800-13809 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 13810-13819 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 13820-13829 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13830-13839 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 13840-13849 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 13850-13859 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13860-13869 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 13870-13879 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 13880-13889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13890-13899 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 13900-13909 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 13910-13919 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 13920-13929 + 1, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 13930-13939 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 13940-13949 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 13950-13959 + 3, 2, 1, 4, 3, 2, 1, 30, 29, 28, // 13960-13969 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 13970-13979 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 13980-13989 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 13990-13999 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 14000-14009 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 14010-14019 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 14020-14029 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 14030-14039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14040-14049 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 14050-14059 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14060-14069 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14070-14079 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 14080-14089 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 14090-14099 + 7, 6, 5, 4, 3, 2, 1, 36, 35, 34, // 14100-14109 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 14110-14119 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 14120-14129 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14130-14139 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 14140-14149 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 14150-14159 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14160-14169 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 14170-14179 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 14180-14189 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 14190-14199 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 14200-14209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14210-14219 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 14220-14229 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14230-14239 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 14240-14249 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 14250-14259 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 14260-14269 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14270-14279 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14280-14289 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 14290-14299 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 14300-14309 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14310-14319 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 14320-14329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14330-14339 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 14340-14349 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 14350-14359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 14360-14369 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 14370-14379 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 14380-14389 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14390-14399 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 14400-14409 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 14410-14419 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 14420-14429 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 14430-14439 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 14440-14449 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14450-14459 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 14460-14469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 14470-14479 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 14480-14489 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14490-14499 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 14500-14509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 14510-14519 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14520-14529 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 14530-14539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 14540-14549 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 14550-14559 + 1, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 14560-14569 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 14570-14579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14580-14589 + 1, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 14590-14599 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 14600-14609 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 14610-14619 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 14620-14629 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 14630-14639 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14640-14649 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 14650-14659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 14660-14669 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14670-14679 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 14680-14689 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 14690-14699 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14700-14709 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 14710-14719 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 14720-14729 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 14730-14739 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 14740-14749 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 14750-14759 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 14760-14769 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 14770-14779 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 14780-14789 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 14790-14799 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14800-14809 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 14810-14819 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 14820-14829 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14830-14839 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 14840-14849 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 14850-14859 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 14860-14869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 14870-14879 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 14880-14889 + 1, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 14890-14899 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 14900-14909 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14910-14919 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 14920-14929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 14930-14939 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 14940-14949 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 14950-14959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 14960-14969 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 14970-14979 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 14980-14989 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 14990-14999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 15000-15009 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 15010-15019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15020-15029 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 15030-15039 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 15040-15049 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 15050-15059 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 15060-15069 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 15070-15079 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 15080-15089 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15090-15099 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 15100-15109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15110-15119 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15120-15129 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 15130-15139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 15140-15149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15150-15159 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 15160-15169 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 15170-15179 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15180-15189 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 15190-15199 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 15200-15209 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 15210-15219 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15220-15229 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 15230-15239 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 15240-15249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 15250-15259 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 15260-15269 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 15270-15279 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 15280-15289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 15290-15299 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15300-15309 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 15310-15319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 15320-15329 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 15330-15339 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 15340-15349 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 15350-15359 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 15360-15369 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 15370-15379 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 15380-15389 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15390-15399 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 15400-15409 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 15410-15419 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 15420-15429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 15430-15439 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 15440-15449 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15450-15459 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15460-15469 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 15470-15479 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 15480-15489 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 15490-15499 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15500-15509 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 15510-15519 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 15520-15529 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15530-15539 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15540-15549 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 15550-15559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 15560-15569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15570-15579 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 15580-15589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15590-15599 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 15600-15609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 15610-15619 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 15620-15629 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15630-15639 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 15640-15649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15650-15659 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 15660-15669 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 15670-15679 + 3, 2, 1, 44, 43, 42, 41, 40, 39, 38, // 15680-15689 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 15690-15699 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 15700-15709 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 15710-15719 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 15720-15729 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 15730-15739 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 15740-15749 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15750-15759 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15760-15769 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 15770-15779 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 15780-15789 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15790-15799 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 15800-15809 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15810-15819 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 15820-15829 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 15830-15839 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 15840-15849 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 15850-15859 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 15860-15869 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 15870-15879 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 15880-15889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15890-15899 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 15900-15909 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 15910-15919 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 15920-15929 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 15930-15939 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 15940-15949 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 15950-15959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15960-15969 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 15970-15979 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15980-15989 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 15990-15999 + 1, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 16000-16009 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 16010-16019 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16020-16029 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 16030-16039 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 16040-16049 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 16050-16059 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 16060-16069 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 16070-16079 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 16080-16089 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16090-16099 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 16100-16109 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 16110-16119 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 16120-16129 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 16130-16139 + 1, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 16140-16149 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 16150-16159 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 16160-16169 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16170-16179 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 16180-16189 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 16190-16199 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 16200-16209 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16210-16219 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 16220-16229 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 16230-16239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 16240-16249 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 16250-16259 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16260-16269 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 16270-16279 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 16280-16289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16290-16299 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 16300-16309 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 16310-16319 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16320-16329 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 16330-16339 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 16340-16349 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16350-16359 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 16360-16369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16370-16379 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 16380-16389 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 16390-16399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16400-16409 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 16410-16419 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16420-16429 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 16430-16439 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 16440-16449 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 16450-16459 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 16460-16469 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 16470-16479 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16480-16489 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 16490-16499 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 16500-16509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 16510-16519 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 16520-16529 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 16530-16539 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16540-16549 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 16550-16559 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16560-16569 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 16570-16579 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 16580-16589 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16590-16599 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 16600-16609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 16610-16619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16620-16629 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 16630-16639 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 16640-16649 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 16650-16659 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16660-16669 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 16670-16679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16680-16689 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 16690-16699 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 16700-16709 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 16710-16719 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 16720-16729 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16730-16739 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 16740-16749 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 16750-16759 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 16760-16769 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 16770-16779 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 16780-16789 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 16790-16799 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16800-16809 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16810-16819 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 16820-16829 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16830-16839 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 16840-16849 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 16850-16859 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16860-16869 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 16870-16879 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 16880-16889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16890-16899 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 16900-16909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 16910-16919 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 16920-16929 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16930-16939 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 16940-16949 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 16950-16959 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 16960-16969 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 16970-16979 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 16980-16989 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 16990-16999 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17000-17009 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17010-17019 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 17020-17029 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 17030-17039 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 17040-17049 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 17050-17059 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17060-17069 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 17070-17079 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17080-17089 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 17090-17099 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 17100-17109 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 17110-17119 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 17120-17129 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 17130-17139 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 17140-17149 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 17150-17159 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 17160-17169 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17170-17179 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 17180-17189 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17190-17199 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 17200-17209 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 17210-17219 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17220-17229 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 17230-17239 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17240-17249 + 7, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 17250-17259 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 17260-17269 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 17270-17279 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17280-17289 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 17290-17299 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17300-17309 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 17310-17319 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 17320-17329 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 17330-17339 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17340-17349 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 17350-17359 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17360-17369 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 17370-17379 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 17380-17389 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 17390-17399 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17400-17409 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 17410-17419 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17420-17429 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17430-17439 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 17440-17449 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17450-17459 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 17460-17469 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 17470-17479 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 17480-17489 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 17490-17499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 17500-17509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 17510-17519 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 17520-17529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 17530-17539 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17540-17549 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 17550-17559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 17560-17569 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 17570-17579 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17580-17589 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 17590-17599 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 17600-17609 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17610-17619 + 3, 2, 1, 4, 3, 2, 1, 30, 29, 28, // 17620-17629 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 17630-17639 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17640-17649 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 17650-17659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 17660-17669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17670-17679 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 17680-17689 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17690-17699 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 17700-17709 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 17710-17719 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 17720-17729 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 17730-17739 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 17740-17749 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17750-17759 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 17760-17769 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17770-17779 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 17780-17789 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17790-17799 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 17800-17809 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17810-17819 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 17820-17829 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 17830-17839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17840-17849 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17850-17859 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 17860-17869 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17870-17879 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17880-17889 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 17890-17899 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 17900-17909 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17910-17919 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 17920-17929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 17930-17939 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 17940-17949 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 17950-17959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 17960-17969 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 17970-17979 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 24, // 17980-17989 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 17990-17999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18000-18009 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 18010-18019 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 18020-18029 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18030-18039 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 18040-18049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 18050-18059 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 18060-18069 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 18070-18079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 18080-18089 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 18090-18099 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 18100-18109 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 18110-18119 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 18120-18129 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 18130-18139 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 18140-18149 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 18150-18159 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 18160-18169 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18170-18179 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18180-18189 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 18190-18199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18200-18209 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 18210-18219 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 18220-18229 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 18230-18239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18240-18249 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 18250-18259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 18260-18269 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 18270-18279 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 18280-18289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18290-18299 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 18300-18309 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 18310-18319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 18320-18329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18330-18339 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18340-18349 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 18350-18359 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 18360-18369 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 18370-18379 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 18380-18389 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 18390-18399 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18400-18409 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 18410-18419 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 18420-18429 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 18430-18439 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 18440-18449 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 18450-18459 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 18460-18469 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18470-18479 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18480-18489 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 18490-18499 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 18500-18509 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 18510-18519 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 18520-18529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 18530-18539 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18540-18549 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 18550-18559 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 18560-18569 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18570-18579 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 18580-18589 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 18590-18599 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 18600-18609 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 18610-18619 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 18620-18629 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 18630-18639 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 18640-18649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18650-18659 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18660-18669 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 18670-18679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18680-18689 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18690-18699 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18700-18709 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 18710-18719 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18720-18729 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18730-18739 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 18740-18749 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 18750-18759 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18760-18769 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 18770-18779 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 18780-18789 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 18790-18799 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 18800-18809 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 18810-18819 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 18820-18829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 18830-18839 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 18840-18849 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 18850-18859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 18860-18869 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 18870-18879 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 18880-18889 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 18890-18899 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18900-18909 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 28, // 18910-18919 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 18920-18929 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 18930-18939 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 18940-18949 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 18950-18959 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 18960-18969 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 18970-18979 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 18980-18989 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 18990-18999 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 19000-19009 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 19010-19019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19020-19029 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 19030-19039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19040-19049 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 19050-19059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 19060-19069 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 19070-19079 + 1, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 19080-19089 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 19090-19099 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 19100-19109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19110-19119 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 19120-19129 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 19130-19139 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 19140-19149 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 19150-19159 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 19160-19169 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19170-19179 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 19180-19189 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 19190-19199 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 19200-19209 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 19210-19219 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19220-19229 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 19230-19239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 19240-19249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 19250-19259 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 19260-19269 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 19270-19279 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 19280-19289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19290-19299 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 19300-19309 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 19310-19319 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 19320-19329 + 3, 2, 1, 40, 39, 38, 37, 36, 35, 34, // 19330-19339 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 19340-19349 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 19350-19359 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 19360-19369 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 19370-19379 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 19380-19389 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 19390-19399 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 19400-19409 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 19410-19419 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 19420-19429 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 19430-19439 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 19440-19449 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 19450-19459 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 19460-19469 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 19470-19479 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 19480-19489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19490-19499 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 19500-19509 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 19510-19519 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19520-19529 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19530-19539 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 19540-19549 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 19550-19559 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19560-19569 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 19570-19579 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 19580-19589 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 19590-19599 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 52, // 19600-19609 + 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, // 19610-19619 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 19620-19629 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 19630-19639 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 19640-19649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19650-19659 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 19660-19669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19670-19679 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 19680-19689 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 19690-19699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 19700-19709 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 19710-19719 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 19720-19729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 19730-19739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19740-19749 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 19750-19759 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 19760-19769 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 19770-19779 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 19780-19789 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 19790-19799 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 19800-19809 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 19810-19819 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 19820-19829 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19830-19839 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 19840-19849 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 19850-19859 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 19860-19869 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 19870-19879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 19880-19889 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 19890-19899 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 19900-19909 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 19910-19919 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 19920-19929 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 19930-19939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 19940-19949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19950-19959 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 19960-19969 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 19970-19979 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 19980-19989 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 19990-19999 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20000-20009 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20010-20019 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 20020-20029 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 20030-20039 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 20040-20049 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20050-20059 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 20060-20069 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 20070-20079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 20080-20089 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20090-20099 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 20100-20109 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 20110-20119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 20120-20129 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20130-20139 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 20140-20149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20150-20159 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20160-20169 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 20170-20179 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 20180-20189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20190-20199 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 20200-20209 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 20210-20219 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20220-20229 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 20230-20239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 20240-20249 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20250-20259 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 20260-20269 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 20270-20279 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 20280-20289 + 7, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 20290-20299 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 20300-20309 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20310-20319 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 20320-20329 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 20330-20339 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 20340-20349 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 20350-20359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 20360-20369 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 20370-20379 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 20380-20389 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 20390-20399 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 20400-20409 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 20410-20419 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20420-20429 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20430-20439 + 1, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 20440-20449 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 20450-20459 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 20460-20469 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 20470-20479 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 20480-20489 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 20490-20499 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 20500-20509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20510-20519 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20520-20529 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 20530-20539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 20540-20549 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20550-20559 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 20560-20569 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 20570-20579 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20580-20589 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 20590-20599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20600-20609 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 20610-20619 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 20620-20629 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 20630-20639 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 20640-20649 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20650-20659 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 20660-20669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20670-20679 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20680-20689 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 20690-20699 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 20700-20709 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 20710-20719 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20720-20729 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20730-20739 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 20740-20749 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 20750-20759 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20760-20769 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 20770-20779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 20780-20789 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 20790-20799 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 40, // 20800-20809 + 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 20810-20819 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 20820-20829 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 20830-20839 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 20840-20849 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 20850-20859 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 20860-20869 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 20870-20879 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 20880-20889 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 20890-20899 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 20900-20909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20910-20919 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 20920-20929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 20930-20939 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 20940-20949 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 20950-20959 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 20960-20969 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20970-20979 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 20980-20989 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 20990-20999 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21000-21009 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 21010-21019 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 21020-21029 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 21030-21039 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 21040-21049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 21050-21059 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 21060-21069 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 21070-21079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 21080-21089 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21090-21099 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 21100-21109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21110-21119 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 21120-21129 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 21130-21139 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 21140-21149 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 21150-21159 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 21160-21169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 21170-21179 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 21180-21189 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 21190-21199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21200-21209 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21210-21219 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 21220-21229 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 21230-21239 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 21240-21249 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 21250-21259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 21260-21269 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 21270-21279 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 21280-21289 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 21290-21299 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 21300-21309 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 21310-21319 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 21320-21329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21330-21339 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 21340-21349 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 21350-21359 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 21360-21369 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 21370-21379 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 21380-21389 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 21390-21399 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 21400-21409 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 21410-21419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 21420-21429 + 3, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 21430-21439 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 21440-21449 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 21450-21459 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 21460-21469 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21470-21479 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 21480-21489 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 21490-21499 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 21500-21509 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 21510-21519 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 28, // 21520-21529 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 21530-21539 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 21540-21549 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 21550-21559 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 21560-21569 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 21570-21579 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 21580-21589 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 21590-21599 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21600-21609 + 1, 2, 1, 4, 3, 2, 1, 30, 29, 28, // 21610-21619 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 21620-21629 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 21630-21639 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 21640-21649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21650-21659 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 21660-21669 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 21670-21679 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 21680-21689 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21690-21699 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 21700-21709 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 21710-21719 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 21720-21729 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 21730-21739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21740-21749 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 21750-21759 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 21760-21769 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 21770-21779 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 21780-21789 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 21790-21799 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 21800-21809 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 21810-21819 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 21820-21829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 21830-21839 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21840-21849 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 21850-21859 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 21860-21869 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21870-21879 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 21880-21889 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 21890-21899 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21900-21909 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 21910-21919 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 21920-21929 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 21930-21939 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 21940-21949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21950-21959 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 21960-21969 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 21970-21979 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 21980-21989 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 21990-21999 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 22000-22009 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 22010-22019 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 22020-22029 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 22030-22039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22040-22049 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22050-22059 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 22060-22069 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 22070-22079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22080-22089 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 22090-22099 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 22100-22109 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22110-22119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 22120-22129 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 22130-22139 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 22140-22149 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 22150-22159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22160-22169 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 22170-22179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 22180-22189 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 22190-22199 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 22200-22209 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 22210-22219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 22220-22229 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 22230-22239 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 22240-22249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 22250-22259 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22260-22269 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 22270-22279 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 22280-22289 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22290-22299 + 3, 2, 1, 4, 3, 2, 1, 36, 35, 34, // 22300-22309 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 22310-22319 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 22320-22329 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22330-22339 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 22340-22349 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 22350-22359 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 22360-22369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22370-22379 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22380-22389 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 22390-22399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 22400-22409 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 22410-22419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22420-22429 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 22430-22439 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 22440-22449 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 22450-22459 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 22460-22469 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22470-22479 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 22480-22489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22490-22499 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22500-22509 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 22510-22519 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22520-22529 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22530-22539 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 22540-22549 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 22550-22559 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 22560-22569 + 1, 2, 1, 40, 39, 38, 37, 36, 35, 34, // 22570-22579 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 22580-22589 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 22590-22599 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22600-22609 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 22610-22619 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 22620-22629 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 22630-22639 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 22640-22649 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 22650-22659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 22660-22669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 22670-22679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22680-22689 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 22690-22699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 22700-22709 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 22710-22719 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 22720-22729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 22730-22739 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22740-22749 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 22750-22759 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 22760-22769 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 22770-22779 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 22780-22789 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 22790-22799 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 22800-22809 + 1, 6, 5, 4, 3, 2, 1, 36, 35, 34, // 22810-22819 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 22820-22829 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 22830-22839 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22840-22849 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 22850-22859 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22860-22869 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 22870-22879 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 22880-22889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22890-22899 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 22900-22909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22910-22919 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 22920-22929 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 22930-22939 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 22940-22949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 22950-22959 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 22960-22969 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 22970-22979 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 22980-22989 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 22990-22999 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 23000-23009 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 23010-23019 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 23020-23029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 23030-23039 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23040-23049 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 23050-23059 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 23060-23069 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23070-23079 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 23080-23089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 23090-23099 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23100-23109 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 23110-23119 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23120-23129 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23130-23139 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 23140-23149 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 23150-23159 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 23160-23169 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 23170-23179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 23180-23189 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 23190-23199 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 23200-23209 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23210-23219 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 23220-23229 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 23230-23239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23240-23249 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 23250-23259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 23260-23269 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 23270-23279 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23280-23289 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 23290-23299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23300-23309 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23310-23319 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 23320-23329 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 23330-23339 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23340-23349 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 23350-23359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 23360-23369 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 23370-23379 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 23380-23389 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 23390-23399 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23400-23409 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 23410-23419 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23420-23429 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23430-23439 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 23440-23449 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 23450-23459 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23460-23469 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 23470-23479 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23480-23489 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 23490-23499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 23500-23509 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 23510-23519 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23520-23529 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 23530-23539 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 23540-23549 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 23550-23559 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 23560-23569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23570-23579 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23580-23589 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 23590-23599 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 23600-23609 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23610-23619 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 23620-23629 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 23630-23639 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 23640-23649 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23650-23659 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 23660-23669 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 23670-23679 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 30, // 23680-23689 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 23690-23699 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 23700-23709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 23710-23719 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 23720-23729 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23730-23739 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 23740-23749 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 23750-23759 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 23760-23769 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 23770-23779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 23780-23789 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23790-23799 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23800-23809 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 23810-23819 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 23820-23829 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 23830-23839 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23840-23849 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 23850-23859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 23860-23869 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 23870-23879 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 23880-23889 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 23890-23899 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 23900-23909 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 23910-23919 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 28, // 23920-23929 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 23930-23939 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 23940-23949 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 23950-23959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 23960-23969 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 23970-23979 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 23980-23989 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 23990-23999 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 24000-24009 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 24010-24019 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 24020-24029 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24030-24039 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 24040-24049 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24050-24059 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24060-24069 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 24070-24079 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 24080-24089 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 24090-24099 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 24100-24109 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 24110-24119 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24120-24129 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 24130-24139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24140-24149 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 24150-24159 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 24160-24169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 24170-24179 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 24180-24189 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 24190-24199 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 24200-24209 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24210-24219 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 24220-24229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 24230-24239 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 24240-24249 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 24250-24259 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 24260-24269 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24270-24279 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 24280-24289 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 24290-24299 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 24300-24309 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 24310-24319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 24320-24329 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 24330-24339 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 24340-24349 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 24350-24359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24360-24369 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 24370-24379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24380-24389 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 24390-24399 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 24400-24409 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 24410-24419 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 24420-24429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 24430-24439 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 24440-24449 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 24450-24459 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 24460-24469 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 24470-24479 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 24480-24489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 24490-24499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 24500-24509 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 24510-24519 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 24520-24529 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 24530-24539 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 24540-24549 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 24550-24559 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24560-24569 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 24570-24579 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24580-24589 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 24590-24599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24600-24609 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24610-24619 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 24620-24629 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 24630-24639 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 24640-24649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 24650-24659 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24660-24669 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 24670-24679 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 24680-24689 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 24690-24699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 24700-24709 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 24710-24719 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24720-24729 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 24730-24739 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 24740-24749 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24750-24759 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 24760-24769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24770-24779 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24780-24789 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 24790-24799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 24800-24809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24810-24819 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 24820-24829 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 24830-24839 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 24840-24849 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 24850-24859 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 24860-24869 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 24870-24879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 24880-24889 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 24890-24899 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 24900-24909 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 24910-24919 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 24920-24929 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 24930-24939 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 24940-24949 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 24950-24959 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 24960-24969 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 24970-24979 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 24980-24989 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 24990-24999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25000-25009 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 25010-25019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25020-25029 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 25030-25039 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 25040-25049 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 25050-25059 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25060-25069 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 25070-25079 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 25080-25089 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 25090-25099 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25100-25109 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 25110-25119 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 25120-25129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 25130-25139 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 25140-25149 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 25150-25159 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 25160-25169 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25170-25179 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 30, // 25180-25189 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 25190-25199 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 25200-25209 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 25210-25219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 25220-25229 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 25230-25239 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 25240-25249 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 25250-25259 + 1, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 25260-25269 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 25270-25279 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 25280-25289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25290-25299 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 25300-25309 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25310-25319 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 25320-25329 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 25330-25339 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 25340-25349 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 25350-25359 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 25360-25369 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 25370-25379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25380-25389 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 25390-25399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 25400-25409 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25410-25419 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 25420-25429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 25430-25439 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 25440-25449 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 25450-25459 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 25460-25469 + 1, 52, 51, 50, 49, 48, 47, 46, 45, 44, // 25470-25479 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 25480-25489 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 25490-25499 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 25500-25509 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25510-25519 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 25520-25529 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 25530-25539 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 25540-25549 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25550-25559 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 25560-25569 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 25570-25579 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 25580-25589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25590-25599 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 25600-25609 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25610-25619 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25620-25629 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 25630-25639 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 25640-25649 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 25650-25659 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 25660-25669 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 25670-25679 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25680-25689 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 25690-25699 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 25700-25709 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 25710-25719 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25720-25729 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 25730-25739 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 25740-25749 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 25750-25759 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 25760-25769 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 25770-25779 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25780-25789 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 25790-25799 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 25800-25809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 25810-25819 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 25820-25829 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25830-25839 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 25840-25849 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 25850-25859 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 25860-25869 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 25870-25879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 25880-25889 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 25890-25899 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 25900-25909 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 25910-25919 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25920-25929 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 25930-25939 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 25940-25949 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 25950-25959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 25960-25969 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 25970-25979 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 25980-25989 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 25990-25999 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 26000-26009 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 26010-26019 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 26020-26029 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26030-26039 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26040-26049 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 26050-26059 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 26060-26069 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26070-26079 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 26080-26089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 26090-26099 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 26100-26109 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 26110-26119 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 26120-26129 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26130-26139 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26140-26149 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 26150-26159 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26160-26169 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 26170-26179 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 26180-26189 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26190-26199 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 26200-26209 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 26210-26219 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 26220-26229 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 26230-26239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 26240-26249 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26250-26259 + 1, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 26260-26269 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 26270-26279 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26280-26289 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 26290-26299 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 26300-26309 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 26310-26319 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 26320-26329 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 26330-26339 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 26340-26349 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 26350-26359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26360-26369 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 26370-26379 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 26380-26389 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 26390-26399 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 26400-26409 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 26410-26419 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 26420-26429 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 26430-26439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 26440-26449 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 26450-26459 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 26460-26469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 26470-26479 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 26480-26489 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 26490-26499 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26500-26509 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 26510-26519 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 26520-26529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 26530-26539 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 26540-26549 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 26550-26559 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26560-26569 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 26570-26579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26580-26589 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 26590-26599 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 26600-26609 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 26610-26619 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 26620-26629 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 26630-26639 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 26640-26649 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 26650-26659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 26660-26669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26670-26679 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 26680-26689 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 26690-26699 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26700-26709 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 26710-26719 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 26720-26729 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 26730-26739 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 26740-26749 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 26750-26759 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 26760-26769 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 26770-26779 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 26780-26789 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26790-26799 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26800-26809 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 26810-26819 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 26820-26829 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 26830-26839 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 26840-26849 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26850-26859 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 26860-26869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 26870-26879 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26880-26889 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 26890-26899 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 26900-26909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26910-26919 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 26920-26929 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 26930-26939 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 26940-26949 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 26950-26959 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 26960-26969 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 26970-26979 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 26980-26989 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 26990-26999 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27000-27009 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 27010-27019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27020-27029 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27030-27039 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 27040-27049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 27050-27059 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 27060-27069 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 27070-27079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27080-27089 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27090-27099 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 18, // 27100-27109 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 27110-27119 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 27120-27129 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27130-27139 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 27140-27149 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 27150-27159 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 27160-27169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 27170-27179 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27180-27189 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 27190-27199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27200-27209 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 27210-27219 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 27220-27229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 27230-27239 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27240-27249 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 27250-27259 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27260-27269 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 27270-27279 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 27280-27289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 27290-27299 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 27300-27309 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 27310-27319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 27320-27329 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 27330-27339 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 27340-27349 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27350-27359 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 27360-27369 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 27370-27379 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 27380-27389 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 27390-27399 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 27400-27409 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 27410-27419 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 27420-27429 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 27430-27439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 27440-27449 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 27450-27459 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 27460-27469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 27470-27479 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 27480-27489 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 27490-27499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 27500-27509 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 27510-27519 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 27520-27529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 27530-27539 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27540-27549 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 27550-27559 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 27560-27569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27570-27579 + 1, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 27580-27589 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 27590-27599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27600-27609 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 27610-27619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27620-27629 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 27630-27639 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 27640-27649 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 27650-27659 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27660-27669 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 27670-27679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 27680-27689 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 27690-27699 + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 27700-27709 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 27710-27719 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27720-27729 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 27730-27739 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 27740-27749 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27750-27759 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 27760-27769 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 27770-27779 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27780-27789 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 27790-27799 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 27800-27809 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 27810-27819 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 27820-27829 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 27830-27839 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 27840-27849 + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 27850-27859 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 27860-27869 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27870-27879 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 27880-27889 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 27890-27899 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 27900-27909 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 27910-27919 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 27920-27929 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 27930-27939 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 27940-27949 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 27950-27959 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 27960-27969 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 27970-27979 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 27980-27989 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 27990-27999 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 28000-28009 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 28010-28019 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 28020-28029 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 28030-28039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28040-28049 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 28050-28059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 28060-28069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28070-28079 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 28080-28089 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 28090-28099 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 28100-28109 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 28110-28119 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 28120-28129 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 28130-28139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28140-28149 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 28150-28159 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 28160-28169 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28170-28179 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 28180-28189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28190-28199 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28200-28209 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 28210-28219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 48, // 28220-28229 + 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, // 28230-28239 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 28240-28249 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 28250-28259 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 28260-28269 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 28270-28279 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 28280-28289 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 28290-28299 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 28300-28309 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 28310-28319 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 28320-28329 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 28330-28339 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 28340-28349 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 28350-28359 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 28360-28369 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 28370-28379 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 28380-28389 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 28390-28399 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 28400-28409 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 28410-28419 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 28420-28429 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 28430-28439 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 28440-28449 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 28450-28459 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 28460-28469 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 28470-28479 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 28480-28489 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 28490-28499 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 28500-28509 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 28510-28519 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 28520-28529 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 28530-28539 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 28540-28549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 28550-28559 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28560-28569 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 28570-28579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28580-28589 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 28590-28599 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 28600-28609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 28610-28619 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 28620-28629 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 28630-28639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 28640-28649 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 28650-28659 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 28660-28669 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 28670-28679 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 28680-28689 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 28690-28699 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 28700-28709 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 28710-28719 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 28720-28729 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 28730-28739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28740-28749 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 28750-28759 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28760-28769 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 28770-28779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 28780-28789 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 28790-28799 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 28800-28809 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 28810-28819 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 28820-28829 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 28830-28839 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 28840-28849 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 28850-28859 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 28860-28869 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 28870-28879 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 28880-28889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28890-28899 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 28900-28909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28910-28919 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 28920-28929 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 28930-28939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 28940-28949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 28950-28959 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 28960-28969 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 28970-28979 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 28980-28989 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 28990-28999 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 29000-29009 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 29010-29019 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 29020-29029 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 29030-29039 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 29040-29049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 29050-29059 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 29060-29069 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 29070-29079 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 29080-29089 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29090-29099 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 29100-29109 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29110-29119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 29120-29129 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 29130-29139 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 29140-29149 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 29150-29159 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 29160-29169 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 29170-29179 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29180-29189 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29190-29199 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 29200-29209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29210-29219 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29220-29229 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29230-29239 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 29240-29249 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 29250-29259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 29260-29269 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 29270-29279 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 29280-29289 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 29290-29299 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 29300-29309 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 29310-29319 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 29320-29329 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 29330-29339 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 29340-29349 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29350-29359 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 29360-29369 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29370-29379 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 29380-29389 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 29390-29399 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29400-29409 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29410-29419 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 29420-29429 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 29430-29439 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 29440-29449 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 29450-29459 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29460-29469 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 29470-29479 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 29480-29489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29490-29499 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 29500-29509 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 29510-29519 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 29520-29529 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 29530-29539 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 29540-29549 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 29550-29559 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 29560-29569 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 29570-29579 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 29580-29589 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 29590-29599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29600-29609 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 29610-29619 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 29620-29629 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 29630-29639 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 29640-29649 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29650-29659 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 29660-29669 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29670-29679 + 3, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 29680-29689 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 29690-29699 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 29700-29709 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 29710-29719 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 29720-29729 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29730-29739 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29740-29749 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 29750-29759 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 29760-29769 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 29770-29779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 29780-29789 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29790-29799 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 29800-29809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 29810-29819 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29820-29829 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 29830-29839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 29840-29849 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29850-29859 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 29860-29869 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 29870-29879 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 29880-29889 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 29890-29899 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 29900-29909 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 29910-29919 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 29920-29929 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 29930-29939 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 29940-29949 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 29950-29959 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 29960-29969 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 29970-29979 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 29980-29989 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 29990-29999 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30000-30009 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 30010-30019 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 30020-30029 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 30030-30039 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 30040-30049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 30050-30059 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30060-30069 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 30070-30079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 30080-30089 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 30090-30099 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 30100-30109 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 30110-30119 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30120-30129 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 30130-30139 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 30140-30149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30150-30159 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 30160-30169 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30170-30179 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 30180-30189 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 30190-30199 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 30200-30209 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30210-30219 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 30220-30229 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30230-30239 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30240-30249 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 30250-30259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 30260-30269 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 30270-30279 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30280-30289 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 30290-30299 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 30300-30309 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 30310-30319 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 30320-30329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30330-30339 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 30340-30349 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 30350-30359 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 30360-30369 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 30370-30379 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 30380-30389 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30390-30399 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 30400-30409 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 30410-30419 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 30420-30429 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 30430-30439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 30440-30449 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 30450-30459 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 30460-30469 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 30470-30479 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30480-30489 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 30490-30499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 30500-30509 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 30510-30519 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 30520-30529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 30530-30539 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30540-30549 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 18, // 30550-30559 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 30560-30569 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 30570-30579 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30580-30589 + 3, 2, 1, 38, 37, 36, 35, 34, 33, 32, // 30590-30599 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 30600-30609 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 30610-30619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30620-30629 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 30630-30639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 30640-30649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30650-30659 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30660-30669 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 30670-30679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 30680-30689 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 30690-30699 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 30700-30709 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 30710-30719 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 30720-30729 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 30730-30739 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 30740-30749 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 30750-30759 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 30760-30769 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 30770-30779 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 30780-30789 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30790-30799 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 30800-30809 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 30810-30819 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 30820-30829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 30830-30839 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30840-30849 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 30850-30859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 30860-30869 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30870-30879 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 30880-30889 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 30890-30899 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30900-30909 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 30910-30919 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30920-30929 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 30930-30939 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 30940-30949 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 30950-30959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 30960-30969 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 30970-30979 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 30980-30989 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 30990-30999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 31000-31009 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 31010-31019 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 31020-31029 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 31030-31039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31040-31049 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 31050-31059 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 31060-31069 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 31070-31079 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31080-31089 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 31090-31099 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 31100-31109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31110-31119 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 31120-31129 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 31130-31139 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 31140-31149 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 31150-31159 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31160-31169 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 31170-31179 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 31180-31189 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 31190-31199 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 31200-31209 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 31210-31219 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 31220-31229 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 31230-31239 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 31240-31249 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 31250-31259 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 31260-31269 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 31270-31279 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 31280-31289 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31290-31299 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 31300-31309 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 31310-31319 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 31320-31329 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 31330-31339 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31340-31349 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 31350-31359 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 31360-31369 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 31370-31379 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 31380-31389 + 1, 2, 1, 4, 3, 2, 1, 72, 71, 70, // 31390-31399 + 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, // 31400-31409 + 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, // 31410-31419 + 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, // 31420-31429 + 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 31430-31439 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 31440-31449 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 31450-31459 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 31460-31469 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 31470-31479 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 31480-31489 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 31490-31499 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31500-31509 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 31510-31519 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31520-31529 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31530-31539 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 31540-31549 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31550-31559 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 31560-31569 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 31570-31579 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 31580-31589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31590-31599 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 31600-31609 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31610-31619 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 31620-31629 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 31630-31639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 31640-31649 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 31650-31659 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 31660-31669 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31670-31679 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 31680-31689 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 31690-31699 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 31700-31709 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31710-31719 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 31720-31729 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31730-31739 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31740-31749 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 31750-31759 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 31760-31769 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 31770-31779 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 31780-31789 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 31790-31799 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31800-31809 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 31810-31819 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 31820-31829 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31830-31839 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 31840-31849 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 31850-31859 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 31860-31869 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 31870-31879 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 31880-31889 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31890-31899 + 7, 6, 5, 4, 3, 2, 1, 50, 49, 48, // 31900-31909 + 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, // 31910-31919 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 31920-31929 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 31930-31939 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 31940-31949 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 31950-31959 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 31960-31969 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 31970-31979 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 31980-31989 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 31990-31999 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 32000-32009 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 32010-32019 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 32020-32029 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 32030-32039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32040-32049 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 32050-32059 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 32060-32069 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 32070-32079 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 32080-32089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 32090-32099 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 32100-32109 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 32110-32119 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 32120-32129 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32130-32139 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 32140-32149 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 32150-32159 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32160-32169 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 32170-32179 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 32180-32189 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32190-32199 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 32200-32209 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 32210-32219 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32220-32229 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 32230-32239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32240-32249 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 32250-32259 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 32260-32269 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 32270-32279 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 32280-32289 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 32290-32299 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 32300-32309 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32310-32319 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 32320-32329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32330-32339 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32340-32349 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 32350-32359 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 32360-32369 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 32370-32379 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 32380-32389 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32390-32399 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32400-32409 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 32410-32419 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 32420-32429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32430-32439 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 32440-32449 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 32450-32459 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 32460-32469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 32470-32479 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32480-32489 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 32490-32499 + 3, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 32500-32509 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 32510-32519 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32520-32529 + 1, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 32530-32539 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 32540-32549 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32550-32559 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 32560-32569 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 32570-32579 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 32580-32589 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32590-32599 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 32600-32609 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32610-32619 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32620-32629 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 32630-32639 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 32640-32649 + 3, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 32650-32659 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 32660-32669 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 32670-32679 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 32680-32689 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 32690-32699 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 32700-32709 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 30, // 32710-32719 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 32720-32729 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 32730-32739 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 32740-32749 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 32750-32759 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32760-32769 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 32770-32779 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 32780-32789 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 32790-32799 + 1, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 32800-32809 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 32810-32819 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 32820-32829 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 32830-32839 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 32840-32849 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 32850-32859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 32860-32869 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 32870-32879 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 32880-32889 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 32890-32899 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 32900-32909 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 32910-32919 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32920-32929 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 32930-32939 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 32940-32949 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 32950-32959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 32960-32969 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 32970-32979 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 32980-32989 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 32990-32999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33000-33009 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 33010-33019 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 33020-33029 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 33030-33039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 33040-33049 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 33050-33059 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33060-33069 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 33070-33079 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 33080-33089 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 33090-33099 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 33100-33109 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 30, // 33110-33119 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 33120-33129 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 33130-33139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 33140-33149 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33150-33159 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 33160-33169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 33170-33179 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33180-33189 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 33190-33199 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 33200-33209 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33210-33219 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 33220-33229 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 33230-33239 + 7, 6, 5, 4, 3, 2, 1, 40, 39, 38, // 33240-33249 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 33250-33259 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 33260-33269 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 33270-33279 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 33280-33289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33290-33299 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33300-33309 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 33310-33319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 33320-33329 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33330-33339 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 33340-33349 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 33350-33359 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 33360-33369 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 33370-33379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33380-33389 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33390-33399 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 33400-33409 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 33410-33419 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 33420-33429 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 33430-33439 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 33440-33449 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 33450-33459 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 33460-33469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 33470-33479 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 33480-33489 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 33490-33499 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 33500-33509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33510-33519 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 33520-33529 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 33530-33539 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 33540-33549 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33550-33559 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 33560-33569 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 33570-33579 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 33580-33589 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 33590-33599 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33600-33609 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 33610-33619 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 33620-33629 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 33630-33639 + 1, 6, 5, 4, 3, 2, 1, 32, 31, 30, // 33640-33649 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 33650-33659 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 33660-33669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 33670-33679 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 33680-33689 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33690-33699 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 33700-33709 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 33710-33719 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 33720-33729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 33730-33739 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 33740-33749 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 33750-33759 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 33760-33769 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 33770-33779 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33780-33789 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 33790-33799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 33800-33809 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 33810-33819 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 33820-33829 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 33830-33839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33840-33849 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 33850-33859 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 33860-33869 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 33870-33879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 33880-33889 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 33890-33899 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33900-33909 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 33910-33919 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 33920-33929 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 33930-33939 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 33940-33949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 33950-33959 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 33960-33969 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 33970-33979 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 33980-33989 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 33990-33999 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 34000-34009 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 34010-34019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34020-34029 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 34030-34039 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 34040-34049 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 34050-34059 + 1, 62, 61, 60, 59, 58, 57, 56, 55, 54, // 34060-34069 + 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, // 34070-34079 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 34080-34089 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 34090-34099 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 34100-34109 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34110-34119 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 34120-34129 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34130-34139 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 34140-34149 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 34150-34159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34160-34169 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34170-34179 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 34180-34189 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 34190-34199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34200-34209 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 34210-34219 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34220-34229 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 34230-34239 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34240-34249 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 34250-34259 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 34260-34269 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 34270-34279 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 34280-34289 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 34290-34299 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 34300-34309 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 34310-34319 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 34320-34329 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 34330-34339 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34340-34349 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34350-34359 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 34360-34369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34370-34379 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 34380-34389 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34390-34399 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 34400-34409 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34410-34419 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 34420-34429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 34430-34439 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 34440-34449 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 34450-34459 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 34460-34469 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34470-34479 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 34480-34489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 34490-34499 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34500-34509 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 34510-34519 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 34520-34529 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 34530-34539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 34, // 34540-34549 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 34550-34559 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 34560-34569 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34570-34579 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 34580-34589 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34590-34599 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 34600-34609 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 34610-34619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34620-34629 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 34630-34639 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 34640-34649 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 34650-34659 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 34660-34669 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 34670-34679 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 34680-34689 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 34690-34699 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 34700-34709 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34710-34719 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 34720-34729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 34730-34739 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 34740-34749 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 34750-34759 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 34760-34769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34770-34779 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 34780-34789 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 34790-34799 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 34800-34809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 34810-34819 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 34820-34829 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34830-34839 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 34840-34849 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 34850-34859 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34860-34869 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 34870-34879 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 34880-34889 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 34890-34899 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 34900-34909 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 34910-34919 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 34920-34929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 34930-34939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 34940-34949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34950-34959 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 34960-34969 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 34970-34979 + 1, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 34980-34989 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 34990-34999 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 35000-35009 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 35010-35019 + 3, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 35020-35029 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35030-35039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35040-35049 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 35050-35059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 35060-35069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35070-35079 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 35080-35089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 35090-35099 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 35100-35109 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 35110-35119 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 35120-35129 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35130-35139 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 35140-35149 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 35150-35159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35160-35169 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 35170-35179 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35180-35189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35190-35199 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35200-35209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35210-35219 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 35220-35229 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35230-35239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35240-35249 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 35250-35259 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 35260-35269 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 35270-35279 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35280-35289 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35290-35299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35300-35309 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 35310-35319 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 35320-35329 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 35330-35339 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 35340-35349 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 35350-35359 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 35360-35369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35370-35379 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 35380-35389 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 35390-35399 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 35400-35409 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 35410-35419 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 35420-35429 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 35430-35439 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 35440-35449 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35450-35459 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 35460-35469 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35470-35479 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35480-35489 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 35490-35499 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 35500-35509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35510-35519 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 35520-35529 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 35530-35539 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 35540-35549 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 35550-35559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 35560-35569 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 35570-35579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35580-35589 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 35590-35599 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 35600-35609 + 7, 6, 5, 4, 3, 2, 1, 54, 53, 52, // 35610-35619 + 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, // 35620-35629 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 35630-35639 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 35640-35649 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35650-35659 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35660-35669 + 1, 6, 5, 4, 3, 2, 1, 52, 51, 50, // 35670-35679 + 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, // 35680-35689 + 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 35690-35699 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 35700-35709 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 35710-35719 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 35720-35729 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 35730-35739 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 35740-35749 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 35750-35759 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35760-35769 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 35770-35779 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 35780-35789 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 35790-35799 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 35800-35809 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 35810-35819 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35820-35829 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 35830-35839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35840-35849 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 35850-35859 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 35860-35869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 35870-35879 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 35880-35889 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 35890-35899 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35900-35909 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 35910-35919 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 35920-35929 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 35930-35939 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 35940-35949 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 35950-35959 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 35960-35969 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 35970-35979 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 35980-35989 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 35990-35999 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 36000-36009 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 36010-36019 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 36020-36029 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 36030-36039 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 36040-36049 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36050-36059 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 36060-36069 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 36070-36079 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 36080-36089 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 36090-36099 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 36100-36109 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 36110-36119 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36120-36129 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 36130-36139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36140-36149 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36150-36159 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 36160-36169 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 36170-36179 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 36180-36189 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 36190-36199 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 36200-36209 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 36210-36219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 36220-36229 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36230-36239 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36240-36249 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36250-36259 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 36260-36269 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 36270-36279 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36280-36289 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 36290-36299 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 36300-36309 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 36310-36319 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 36320-36329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36330-36339 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 36340-36349 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 36350-36359 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36360-36369 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 36370-36379 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 44, // 36380-36389 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 36390-36399 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 36400-36409 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 36410-36419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36420-36429 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 36430-36439 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36440-36449 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 36450-36459 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 36460-36469 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 36470-36479 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36480-36489 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 36490-36499 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 36500-36509 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36510-36519 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 36520-36529 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36530-36539 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36540-36549 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 36550-36559 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 36560-36569 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36570-36579 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 36580-36589 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 36590-36599 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 36600-36609 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 36610-36619 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 36620-36629 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 36630-36639 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 36640-36649 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 36650-36659 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36660-36669 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 36670-36679 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 36680-36689 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 36690-36699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 36700-36709 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 36710-36719 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 36720-36729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 36730-36739 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 36740-36749 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36750-36759 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 36760-36769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 36770-36779 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 36780-36789 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 36790-36799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 36800-36809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36810-36819 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36820-36829 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 36830-36839 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 36840-36849 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 36850-36859 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 36860-36869 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 36870-36879 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 36880-36889 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 36890-36899 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36900-36909 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 36910-36919 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 36920-36929 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36930-36939 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 36940-36949 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 36950-36959 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 36960-36969 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 36970-36979 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 36980-36989 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 36990-36999 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 37000-37009 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 37010-37019 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 37020-37029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 37030-37039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 37040-37049 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 37050-37059 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 37060-37069 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37070-37079 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 37080-37089 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 37090-37099 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37100-37109 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 37110-37119 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 37120-37129 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 37130-37139 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 37140-37149 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 37150-37159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37160-37169 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37170-37179 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 37180-37189 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 37190-37199 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37200-37209 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 37210-37219 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 37220-37229 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 37230-37239 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 37240-37249 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 37250-37259 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 37260-37269 + 3, 2, 1, 4, 3, 2, 1, 30, 29, 28, // 37270-37279 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 37280-37289 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37290-37299 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 37300-37309 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 37310-37319 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37320-37329 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 37330-37339 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37340-37349 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 37350-37359 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 37360-37369 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 37370-37379 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37380-37389 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 37390-37399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 37400-37409 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 37410-37419 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 37420-37429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37430-37439 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 37440-37449 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 37450-37459 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 37460-37469 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 37470-37479 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 37480-37489 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 37490-37499 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 37500-37509 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 37510-37519 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 37520-37529 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 37530-37539 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 37540-37549 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37550-37559 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 37560-37569 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 37570-37579 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 37580-37589 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37590-37599 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 37600-37609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 37610-37619 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 37620-37629 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 37630-37639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 37640-37649 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 37650-37659 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 37660-37669 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 37670-37679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37680-37689 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 37690-37699 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37700-37709 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 37710-37719 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 37720-37729 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37730-37739 + 7, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 37740-37749 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 37750-37759 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 37760-37769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37770-37779 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 37780-37789 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 37790-37799 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37800-37809 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 37810-37819 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37820-37829 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37830-37839 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 37840-37849 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 37850-37859 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37860-37869 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 37870-37879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 37880-37889 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 37890-37899 + 7, 6, 5, 4, 3, 2, 1, 44, 43, 42, // 37900-37909 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 37910-37919 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 37920-37929 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 37930-37939 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 37940-37949 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 37950-37959 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 37960-37969 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 37970-37979 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 37980-37989 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 37990-37999 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38000-38009 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 38010-38019 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 38020-38029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 38030-38039 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 38040-38049 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 38050-38059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 38060-38069 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38070-38079 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 38080-38089 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 38090-38099 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38100-38109 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 30, // 38110-38119 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 38120-38129 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 38130-38139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 38140-38149 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 38150-38159 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 38160-38169 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 38170-38179 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 38180-38189 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 38190-38199 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 38200-38209 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 38210-38219 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38220-38229 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 38230-38239 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 38240-38249 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38250-38259 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38260-38269 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 38270-38279 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 38280-38289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 38290-38299 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 38300-38309 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 38310-38319 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 38320-38329 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 38330-38339 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38340-38349 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 38350-38359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38360-38369 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 38370-38379 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38380-38389 + 3, 2, 1, 38, 37, 36, 35, 34, 33, 32, // 38390-38399 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 38400-38409 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 38410-38419 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38420-38429 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 38430-38439 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 38440-38449 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 38450-38459 + 1, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 38460-38469 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 38470-38479 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 38480-38489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38490-38499 + 1, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 38500-38509 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 38510-38519 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 38520-38529 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38530-38539 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 38540-38549 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 38550-38559 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 24, // 38560-38569 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 38570-38579 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38580-38589 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 38590-38599 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 38600-38609 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 38610-38619 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 38620-38629 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 38630-38639 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38640-38649 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 38650-38659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 38660-38669 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 38670-38679 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38680-38689 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 38690-38699 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 38700-38709 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 38710-38719 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 38720-38729 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 38730-38739 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 38740-38749 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 38750-38759 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 38760-38769 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38770-38779 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 38780-38789 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38790-38799 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 38800-38809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38810-38819 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38820-38829 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 38830-38839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38840-38849 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38850-38859 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 38860-38869 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 38870-38879 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38880-38889 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38890-38899 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 38900-38909 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 38910-38919 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 38920-38929 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 38930-38939 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38940-38949 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 38950-38959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 38960-38969 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 38970-38979 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 38980-38989 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 38990-38999 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 39000-39009 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 39010-39019 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 39020-39029 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39030-39039 + 1, 2, 1, 4, 3, 2, 1, 32, 31, 30, // 39040-39049 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 39050-39059 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 39060-39069 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 39070-39079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 39080-39089 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 39090-39099 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 39100-39109 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 39110-39119 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 39120-39129 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 39130-39139 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 39140-39149 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 39150-39159 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 39160-39169 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39170-39179 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39180-39189 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 39190-39199 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 39200-39209 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 39210-39219 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 39220-39229 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 39230-39239 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39240-39249 + 1, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 39250-39259 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 39260-39269 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 39270-39279 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 39280-39289 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 39290-39299 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 39300-39309 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 39310-39319 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 39320-39329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39330-39339 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 39340-39349 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 39350-39359 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 39360-39369 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 39370-39379 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 39380-39389 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 39390-39399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 39400-39409 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 39410-39419 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 39420-39429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 39430-39439 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 39440-39449 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39450-39459 + 1, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 39460-39469 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 39470-39479 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 39480-39489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 39490-39499 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 39500-39509 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39510-39519 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 39520-39529 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39530-39539 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39540-39549 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 39550-39559 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 39560-39569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39570-39579 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 39580-39589 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 39590-39599 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 39600-39609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 39610-39619 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 39620-39629 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 39630-39639 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 39640-39649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 39650-39659 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 39660-39669 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 39670-39679 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 39680-39689 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 39690-39699 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 39700-39709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 39710-39719 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 39720-39729 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 39730-39739 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 39740-39749 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39750-39759 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 39760-39769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 39770-39779 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39780-39789 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 39790-39799 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 39800-39809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39810-39819 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 39820-39829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 39830-39839 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 39840-39849 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 39850-39859 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 39860-39869 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 39870-39879 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 39880-39889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39890-39899 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 39900-39909 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 39910-39919 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 39920-39929 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 39930-39939 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 39940-39949 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 39950-39959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 39960-39969 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 39970-39979 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 39980-39989 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 39990-39999 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 40000-40009 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 40010-40019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40020-40029 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 24, // 40030-40039 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 40040-40049 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40050-40059 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 40060-40069 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 40070-40079 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 40080-40089 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 40090-40099 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40100-40109 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40110-40119 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 40120-40129 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 40130-40139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40140-40149 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 40150-40159 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 40160-40169 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 40170-40179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 40180-40189 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 40190-40199 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40200-40209 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 40210-40219 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40220-40229 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 40230-40239 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40240-40249 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 40250-40259 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 40260-40269 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 40270-40279 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 54, // 40280-40289 + 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, // 40290-40299 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 40300-40309 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 40310-40319 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 40320-40329 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40330-40339 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 40340-40349 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 40350-40359 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 40360-40369 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 40370-40379 + 7, 6, 5, 4, 3, 2, 1, 36, 35, 34, // 40380-40389 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 40390-40399 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 40400-40409 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40410-40419 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 40420-40429 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 40430-40439 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 40440-40449 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 40450-40459 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40460-40469 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40470-40479 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 40480-40489 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 40490-40499 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 40500-40509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 40510-40519 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 40520-40529 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40530-40539 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 40540-40549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 40550-40559 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 40560-40569 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 40570-40579 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 40580-40589 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 40590-40599 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 40600-40609 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 40610-40619 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 40620-40629 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 54, // 40630-40639 + 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, // 40640-40649 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 40650-40659 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 40660-40669 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 40670-40679 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40680-40689 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 40690-40699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 40700-40709 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 40710-40719 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 40720-40729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 40730-40739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40740-40749 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 40750-40759 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 40760-40769 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 40770-40779 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 40780-40789 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40790-40799 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40800-40809 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 40810-40819 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 40820-40829 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40830-40839 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 40840-40849 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 40850-40859 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 40860-40869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 40870-40879 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 40880-40889 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 40890-40899 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 40900-40909 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 40910-40919 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 40920-40929 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 40930-40939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 40940-40949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 40950-40959 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40960-40969 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 40970-40979 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 40980-40989 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 40990-40999 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41000-41009 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 41010-41019 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 41020-41029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 41030-41039 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 41040-41049 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 41050-41059 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 41060-41069 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 41070-41079 + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 41080-41089 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 41090-41099 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 41100-41109 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 41110-41119 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41120-41129 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41130-41139 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 41140-41149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41150-41159 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 41160-41169 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 41170-41179 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 41180-41189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41190-41199 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 41200-41209 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 41210-41219 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 41220-41229 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 41230-41239 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 41240-41249 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 41250-41259 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 41260-41269 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41270-41279 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 41280-41289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 34, // 41290-41299 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 41300-41309 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 41310-41319 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 41320-41329 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 41330-41339 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41340-41349 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 41350-41359 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 41360-41369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41370-41379 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 41380-41389 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 41390-41399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41400-41409 + 1, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 41410-41419 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 41420-41429 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 41430-41439 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 41440-41449 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 41450-41459 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 41460-41469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 41470-41479 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41480-41489 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 41490-41499 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 41500-41509 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 41510-41519 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 41520-41529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 41530-41539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 30, // 41540-41549 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 41550-41559 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 41560-41569 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 41570-41579 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 41580-41589 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 41590-41599 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 41600-41609 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 41610-41619 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 41620-41629 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41630-41639 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 41640-41649 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 41650-41659 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 41660-41669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41670-41679 + 1, 6, 5, 4, 3, 2, 1, 32, 31, 30, // 41680-41689 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 41690-41699 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 41700-41709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 41710-41719 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 41720-41729 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 41730-41739 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 41740-41749 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 41750-41759 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41760-41769 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 41770-41779 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 41780-41789 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41790-41799 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 41800-41809 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 41810-41819 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 41820-41829 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 41830-41839 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 41840-41849 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 41850-41859 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 41860-41869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 41870-41879 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 41880-41889 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 41890-41899 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 41900-41909 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 41910-41919 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 41920-41929 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41930-41939 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 41940-41949 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 41950-41959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 41960-41969 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 41970-41979 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 41980-41989 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 41990-41999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42000-42009 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 42010-42019 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 42020-42029 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42030-42039 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 42040-42049 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42050-42059 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42060-42069 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 42070-42079 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 42080-42089 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42090-42099 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 42100-42109 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 42110-42119 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42120-42129 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 42130-42139 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 42140-42149 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 42150-42159 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 42160-42169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 42170-42179 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 42180-42189 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 42190-42199 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 42200-42209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42210-42219 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 42220-42229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 42230-42239 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 42240-42249 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 42250-42259 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 42260-42269 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42270-42279 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 42280-42289 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 42290-42299 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 42300-42309 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42310-42319 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 42320-42329 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 42330-42339 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 42340-42349 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 42350-42359 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42360-42369 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 42370-42379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42380-42389 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 42390-42399 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 24, // 42400-42409 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 42410-42419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42420-42429 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 42430-42439 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 42440-42449 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 42450-42459 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 42460-42469 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 42470-42479 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 42480-42489 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 42490-42499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 42500-42509 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 42510-42519 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42520-42529 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 42530-42539 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 42540-42549 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 42550-42559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 42560-42569 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 42570-42579 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 42580-42589 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 42590-42599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42600-42609 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 42610-42619 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 42620-42629 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42630-42639 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 42640-42649 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 42650-42659 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 42660-42669 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 42670-42679 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 42680-42689 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 42690-42699 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 42700-42709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 42710-42719 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 42720-42729 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 42730-42739 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 42740-42749 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 42750-42759 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 42760-42769 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 42770-42779 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 42780-42789 + 3, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 42790-42799 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 42800-42809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 42810-42819 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 42820-42829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 42830-42839 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42840-42849 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 42850-42859 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 42860-42869 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 42870-42879 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 42880-42889 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 42890-42899 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 42900-42909 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42910-42919 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 42920-42929 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 42930-42939 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 42940-42949 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 42950-42959 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 42960-42969 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 42970-42979 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 42980-42989 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 42990-42999 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 43000-43009 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 43010-43019 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 43020-43029 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 43030-43039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 43040-43049 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43050-43059 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 43060-43069 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 43070-43079 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43080-43089 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 43090-43099 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 43100-43109 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 43110-43119 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43120-43129 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 43130-43139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43140-43149 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 43150-43159 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 43160-43169 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 43170-43179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 43180-43189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43190-43199 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 43200-43209 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43210-43219 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 43220-43229 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 43230-43239 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 43240-43249 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43250-43259 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43260-43269 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43270-43279 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 43280-43289 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 43290-43299 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43300-43309 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 43310-43319 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43320-43329 + 1, 60, 59, 58, 57, 56, 55, 54, 53, 52, // 43330-43339 + 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, // 43340-43349 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 43350-43359 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 43360-43369 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 43370-43379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43380-43389 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 43390-43399 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 43400-43409 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 43410-43419 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 43420-43429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43430-43439 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43440-43449 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 43450-43459 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 43460-43469 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43470-43479 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 43480-43489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 43490-43499 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 43500-43509 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 43510-43519 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 43520-43529 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43530-43539 + 1, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 43540-43549 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 43550-43559 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43560-43569 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 43570-43579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43580-43589 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 43590-43599 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 43600-43609 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 43610-43619 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 43620-43629 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 43630-43639 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 43640-43649 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43650-43659 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 43660-43669 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 43670-43679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43680-43689 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 43690-43699 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43700-43709 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 43710-43719 + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 43720-43729 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 43730-43739 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43740-43749 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 43750-43759 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 43760-43769 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 43770-43779 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 43780-43789 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 43790-43799 + 1, 52, 51, 50, 49, 48, 47, 46, 45, 44, // 43800-43809 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 43810-43819 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 43820-43829 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 43830-43839 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43840-43849 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 43850-43859 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 43860-43869 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 43870-43879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 43880-43889 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 43890-43899 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43900-43909 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 43910-43919 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 43920-43929 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 43930-43939 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 43940-43949 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 43950-43959 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 43960-43969 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 43970-43979 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 43980-43989 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 43990-43999 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 44000-44009 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 44010-44019 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 44020-44029 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44030-44039 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44040-44049 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 44050-44059 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44060-44069 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 44070-44079 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 44080-44089 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44090-44099 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44100-44109 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 44110-44119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 44120-44129 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 44130-44139 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 44140-44149 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 44150-44159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44160-44169 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 44170-44179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 44180-44189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44190-44199 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 44200-44209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44210-44219 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 44220-44229 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 44230-44239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 44240-44249 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 44250-44259 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 44260-44269 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 44270-44279 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44280-44289 + 3, 2, 1, 58, 57, 56, 55, 54, 53, 52, // 44290-44299 + 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, // 44300-44309 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 44310-44319 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 44320-44329 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 44330-44339 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44340-44349 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 44350-44359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44360-44369 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44370-44379 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 28, // 44380-44389 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 44390-44399 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 44400-44409 + 7, 6, 5, 4, 3, 2, 1, 32, 31, 30, // 44410-44419 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 44420-44429 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 44430-44439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 44440-44449 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 44450-44459 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 44460-44469 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44470-44479 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 44480-44489 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 44490-44499 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 44500-44509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 44510-44519 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44520-44529 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 44530-44539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 44540-44549 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44550-44559 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 44560-44569 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 44570-44579 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 44580-44589 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 44590-44599 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 44600-44609 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 44610-44619 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 44620-44629 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 44630-44639 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 44640-44649 + 1, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 44650-44659 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 44660-44669 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44670-44679 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 44680-44689 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 44690-44699 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44700-44709 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 44710-44719 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 44720-44729 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44730-44739 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44740-44749 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 44750-44759 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 44760-44769 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 44770-44779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 44780-44789 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 44790-44799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 44800-44809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 44810-44819 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 44820-44829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 44830-44839 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 44840-44849 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 44850-44859 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 44860-44869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 44870-44879 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 44880-44889 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 44890-44899 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 44900-44909 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 44910-44919 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 44920-44929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 44930-44939 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44940-44949 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 44950-44959 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 44960-44969 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 44970-44979 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 44980-44989 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 44990-44999 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 45000-45009 + 3, 2, 1, 40, 39, 38, 37, 36, 35, 34, // 45010-45019 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 45020-45029 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 45030-45039 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45040-45049 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 45050-45059 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 45060-45069 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 45070-45079 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 45080-45089 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 45090-45099 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 45100-45109 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 45110-45119 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 45120-45129 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 45130-45139 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 45140-45149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45150-45159 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 45160-45169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 45170-45179 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45180-45189 + 1, 6, 5, 4, 3, 2, 1, 36, 35, 34, // 45190-45199 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 45200-45209 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 45210-45219 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45220-45229 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 45230-45239 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 45240-45249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 45250-45259 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 45260-45269 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45270-45279 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 45280-45289 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 45290-45299 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 45300-45309 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 45310-45319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 45320-45329 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 45330-45339 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 45340-45349 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45350-45359 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 45360-45369 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 45370-45379 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 45380-45389 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45390-45399 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 45400-45409 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 45410-45419 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 45420-45429 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 42, // 45430-45439 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 45440-45449 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 45450-45459 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 45460-45469 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45470-45479 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45480-45489 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 45490-45499 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 45500-45509 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45510-45519 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 45520-45529 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 45530-45539 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45540-45549 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 45550-45559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 45560-45569 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 45570-45579 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 45580-45589 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 45590-45599 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45600-45609 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 45610-45619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45620-45629 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45630-45639 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 45640-45649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 45650-45659 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 45660-45669 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 45670-45679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45680-45689 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 45690-45699 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 45700-45709 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 45710-45719 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 45720-45729 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 45730-45739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45740-45749 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 45750-45759 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 45760-45769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 38, // 45770-45779 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 45780-45789 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 45790-45799 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 45800-45809 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 45810-45819 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 45820-45829 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 45830-45839 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45840-45849 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 45850-45859 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 45860-45869 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 45870-45879 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 45880-45889 + 3, 2, 1, 50, 49, 48, 47, 46, 45, 44, // 45890-45899 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 45900-45909 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 45910-45919 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 45920-45929 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 45930-45939 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 45940-45949 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 45950-45959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 45960-45969 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 45970-45979 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 32, // 45980-45989 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 45990-45999 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 46000-46009 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46010-46019 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 46020-46029 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 46030-46039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 46040-46049 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46050-46059 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46060-46069 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 46070-46079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46080-46089 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 46090-46099 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 46100-46109 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 46110-46119 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46120-46129 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 46130-46139 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 46140-46149 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 46150-46159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46160-46169 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46170-46179 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 46180-46189 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 46190-46199 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 46200-46209 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 46210-46219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 46220-46229 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 46230-46239 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 46240-46249 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46250-46259 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46260-46269 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 46270-46279 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 46280-46289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46290-46299 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 46300-46309 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 46310-46319 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 46320-46329 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 46330-46339 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 46340-46349 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 46350-46359 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 46360-46369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46370-46379 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 46380-46389 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 46390-46399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46400-46409 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 46410-46419 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 46420-46429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 46430-46439 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 46440-46449 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 46450-46459 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46460-46469 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 46470-46479 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 46480-46489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 46490-46499 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 46500-46509 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46510-46519 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 46520-46529 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 46530-46539 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 46540-46549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 46550-46559 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 46560-46569 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 46570-46579 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 46580-46589 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46590-46599 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 46600-46609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 46610-46619 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46620-46629 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 46630-46639 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 46640-46649 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46650-46659 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 46660-46669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 46670-46679 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 46680-46689 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46690-46699 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 46700-46709 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46710-46719 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 46720-46729 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 46730-46739 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 46740-46749 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 46750-46759 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 46760-46769 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 46770-46779 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 46780-46789 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 46790-46799 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 46800-46809 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 46810-46819 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 46820-46829 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 46830-46839 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46840-46849 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 46850-46859 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 46860-46869 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 46870-46879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 46880-46889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 46890-46899 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 46900-46909 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 46910-46919 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46920-46929 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 46930-46939 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 46940-46949 + 7, 6, 5, 4, 3, 2, 1, 36, 35, 34, // 46950-46959 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 46960-46969 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 46970-46979 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 46980-46989 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 46990-46999 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47000-47009 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 47010-47019 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 47020-47029 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47030-47039 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47040-47049 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 28, // 47050-47059 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 47060-47069 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47070-47079 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 47080-47089 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 47090-47099 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47100-47109 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 47110-47119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 47120-47129 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 47130-47139 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 47140-47149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47150-47159 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 47160-47169 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 47170-47179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 47180-47189 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47190-47199 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 47200-47209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47210-47219 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47220-47229 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 47230-47239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47240-47249 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 47250-47259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 47260-47269 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 47270-47279 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 47280-47289 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 47290-47299 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 47300-47309 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 47310-47319 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 47320-47329 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 47330-47339 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47340-47349 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 47350-47359 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 47360-47369 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47370-47379 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 47380-47389 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47390-47399 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 47400-47409 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 47410-47419 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47420-47429 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47430-47439 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 47440-47449 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 32, // 47450-47459 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 47460-47469 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 47470-47479 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47480-47489 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 47490-47499 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 47500-47509 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 47510-47519 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 47520-47529 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 47530-47539 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 47540-47549 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 47550-47559 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 47560-47569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47570-47579 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47580-47589 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 47590-47599 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 47600-47609 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 47610-47619 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 47620-47629 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 47630-47639 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 47640-47649 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 47650-47659 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 47660-47669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47670-47679 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 47680-47689 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 47690-47699 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47700-47709 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 47710-47719 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47720-47729 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 47730-47739 + 1, 2, 1, 34, 33, 32, 31, 30, 29, 28, // 47740-47749 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 47750-47759 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47760-47769 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 47770-47779 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47780-47789 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 47790-47799 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 47800-47809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 47810-47819 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 47820-47829 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 47830-47839 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 47840-47849 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 47850-47859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 47860-47869 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 47870-47879 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 47880-47889 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 47890-47899 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 47900-47909 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 47910-47919 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 47920-47929 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 47930-47939 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 47940-47949 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 47950-47959 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 47960-47969 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 47970-47979 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 47980-47989 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 47990-47999 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48000-48009 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 48010-48019 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 48020-48029 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 48030-48039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 48040-48049 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 48050-48059 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48060-48069 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 48070-48079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48080-48089 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 48090-48099 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 48100-48109 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 48110-48119 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48120-48129 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 48130-48139 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48140-48149 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 48150-48159 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 48160-48169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 48170-48179 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 48180-48189 + 3, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 48190-48199 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 48200-48209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48210-48219 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 48220-48229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 48230-48239 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 48240-48249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 48250-48259 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48260-48269 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48270-48279 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 48280-48289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 48290-48299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48300-48309 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 48310-48319 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48320-48329 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 48330-48339 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48340-48349 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 48350-48359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48360-48369 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48370-48379 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 48380-48389 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 48390-48399 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 48400-48409 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 48410-48419 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48420-48429 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 48430-48439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 48440-48449 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48450-48459 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 48460-48469 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 48470-48479 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 48480-48489 + 1, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 48490-48499 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 48500-48509 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48510-48519 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 48520-48529 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 48530-48539 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 48540-48549 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48550-48559 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 48560-48569 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 48570-48579 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 48580-48589 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 48590-48599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48600-48609 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 48610-48619 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 48620-48629 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48630-48639 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 48640-48649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48650-48659 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48660-48669 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 52, // 48670-48679 + 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, // 48680-48689 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 48690-48699 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 48700-48709 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 48710-48719 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48720-48729 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 48730-48739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 48740-48749 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 48750-48759 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 48760-48769 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 48770-48779 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 48780-48789 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 48790-48799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 48800-48809 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 48810-48819 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 48820-48829 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48830-48839 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 48840-48849 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 48850-48859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 48860-48869 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48870-48879 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 48880-48889 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48890-48899 + 7, 6, 5, 4, 3, 2, 1, 40, 39, 38, // 48900-48909 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 48910-48919 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 48920-48929 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 48930-48939 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 48940-48949 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 48950-48959 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48960-48969 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 48970-48979 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 48980-48989 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 48990-48999 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 49000-49009 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 49010-49019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49020-49029 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 49030-49039 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 49040-49049 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 49050-49059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 49060-49069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49070-49079 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 49080-49089 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49090-49099 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 49100-49109 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 49110-49119 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 49120-49129 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 49130-49139 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 49140-49149 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 49150-49159 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 49160-49169 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 49170-49179 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49180-49189 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 49190-49199 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 49200-49209 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49210-49219 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 49220-49229 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 49230-49239 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49240-49249 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 49250-49259 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 49260-49269 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 49270-49279 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 49280-49289 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 49290-49299 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 49300-49309 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 49310-49319 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49320-49329 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 24, // 49330-49339 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 49340-49349 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49350-49359 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 22, // 49360-49369 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 49370-49379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49380-49389 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 49390-49399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 49400-49409 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 49410-49419 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 49420-49429 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 49430-49439 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49440-49449 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 49450-49459 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 49460-49469 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 49470-49479 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 49480-49489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 49490-49499 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 49500-49509 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49510-49519 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 49520-49529 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 49530-49539 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 49540-49549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 38, // 49550-49559 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 49560-49569 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 49570-49579 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 49580-49589 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 49590-49599 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 49600-49609 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 49610-49619 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 49620-49629 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 24, // 49630-49639 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 49640-49649 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49650-49659 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 49660-49669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49670-49679 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 49680-49689 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 49690-49699 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49700-49709 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 49710-49719 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 49720-49729 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 49730-49739 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 49740-49749 + 7, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 49750-49759 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 49760-49769 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49770-49779 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 49780-49789 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49790-49799 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 49800-49809 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49810-49819 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 49820-49829 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 49830-49839 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 49840-49849 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 49850-49859 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49860-49869 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 49870-49879 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49880-49889 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 49890-49899 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 49900-49909 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 49910-49919 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 49920-49929 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 49930-49939 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 49940-49949 + 7, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 49950-49959 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 49960-49969 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 49970-49979 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 49980-49989 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 49990-49999 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50000-50009 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50010-50019 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 50020-50029 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 50030-50039 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 50040-50049 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 50050-50059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 50060-50069 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 50070-50079 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 50080-50089 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 50090-50099 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50100-50109 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 50110-50119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 50120-50129 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 50130-50139 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 50140-50149 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 50150-50159 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 50160-50169 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 50170-50179 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 50180-50189 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 50190-50199 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 50200-50209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50210-50219 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 50220-50229 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 50230-50239 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50240-50249 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50250-50259 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 50260-50269 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 50270-50279 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 50280-50289 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50290-50299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50300-50309 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50310-50319 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 50320-50329 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 50330-50339 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 50340-50349 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 50350-50359 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 50360-50369 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 50370-50379 + 3, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 50380-50389 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50390-50399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50400-50409 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 50410-50419 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 50420-50429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50430-50439 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 50440-50449 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 50450-50459 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 50460-50469 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 50470-50479 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 50480-50489 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 50490-50499 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 50500-50509 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 50510-50519 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 50520-50529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 50530-50539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 50540-50549 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 50550-50559 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50560-50569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50570-50579 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 50580-50589 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 28, // 50590-50599 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 50600-50609 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 50610-50619 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 50620-50629 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 50630-50639 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 50640-50649 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50650-50659 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50660-50669 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 50670-50679 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 50680-50689 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 50690-50699 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 50700-50709 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 50710-50719 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 50720-50729 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50730-50739 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 50740-50749 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 50750-50759 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 50760-50769 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 50770-50779 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 32, // 50780-50789 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 50790-50799 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50800-50809 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50810-50819 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 50820-50829 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 50830-50839 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 50840-50849 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 50850-50859 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 50860-50869 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 50870-50879 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50880-50889 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 50890-50899 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 50900-50909 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 50910-50919 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 50920-50929 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 50930-50939 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 50940-50949 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 50950-50959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 50960-50969 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 50970-50979 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 50980-50989 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 50990-50999 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 51000-51009 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 51010-51019 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51020-51029 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51030-51039 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 51040-51049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 51050-51059 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51060-51069 + 1, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 51070-51079 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 51080-51089 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 51090-51099 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 51100-51109 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 51110-51119 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51120-51129 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 51130-51139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51140-51149 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 51150-51159 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 51160-51169 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 51170-51179 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51180-51189 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 51190-51199 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 51200-51209 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 51210-51219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 51220-51229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 51230-51239 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 51240-51249 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 51250-51259 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 51260-51269 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51270-51279 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 51280-51289 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 51290-51299 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 51300-51309 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 51310-51319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 51320-51329 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51330-51339 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 12, // 51340-51349 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51350-51359 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 51360-51369 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51370-51379 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 51380-51389 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 51390-51399 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 51400-51409 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 51410-51419 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 51420-51429 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 51430-51439 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 51440-51449 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51450-51459 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51460-51469 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 51470-51479 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 51480-51489 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51490-51499 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 51500-51509 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 51510-51519 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 51520-51529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 51530-51539 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51540-51549 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51550-51559 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 51560-51569 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 51570-51579 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51580-51589 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 51590-51599 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 51600-51609 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 51610-51619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51620-51629 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 51630-51639 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 51640-51649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 51650-51659 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51660-51669 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 51670-51679 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 51680-51689 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 51690-51699 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51700-51709 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 51710-51719 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 51720-51729 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 51730-51739 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 51740-51749 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 51750-51759 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 51760-51769 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 51770-51779 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 51780-51789 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 51790-51799 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 51800-51809 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 51810-51819 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 51820-51829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 51830-51839 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51840-51849 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 51850-51859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 51860-51869 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 51870-51879 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 51880-51889 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 51890-51899 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 51900-51909 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 51910-51919 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 51920-51929 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51930-51939 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 51940-51949 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 51950-51959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51960-51969 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 51970-51979 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 51980-51989 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 51990-51999 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 52000-52009 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52010-52019 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 52020-52029 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 52030-52039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52040-52049 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 52050-52059 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 52060-52069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52070-52079 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 52080-52089 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 52090-52099 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 52100-52109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52110-52119 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 52120-52129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 52130-52139 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 52140-52149 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 52150-52159 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 52160-52169 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 52170-52179 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 52180-52189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52190-52199 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 52200-52209 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 52210-52219 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 52220-52229 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 52230-52239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 52240-52249 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 52250-52259 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 52260-52269 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 52270-52279 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 52280-52289 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52290-52299 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 52300-52309 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 52310-52319 + 1, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 52320-52329 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 52330-52339 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 52340-52349 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52350-52359 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 52360-52369 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 52370-52379 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 52380-52389 + 1, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 52390-52399 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 52400-52409 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 52410-52419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 52420-52429 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 52430-52439 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 52440-52449 + 3, 2, 1, 4, 3, 2, 1, 32, 31, 30, // 52450-52459 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 52460-52469 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 52470-52479 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 52480-52489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52490-52499 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52500-52509 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 52510-52519 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 52520-52529 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52530-52539 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 52540-52549 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 52550-52559 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 52560-52569 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 52570-52579 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 52580-52589 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 52590-52599 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 52600-52609 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 52610-52619 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 52620-52629 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 28, // 52630-52639 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 52640-52649 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 52650-52659 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 52660-52669 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 52670-52679 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52680-52689 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 52690-52699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 52700-52709 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52710-52719 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 52720-52729 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 52730-52739 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 52740-52749 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 52750-52759 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 52760-52769 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 52770-52779 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 52780-52789 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 52790-52799 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 52800-52809 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 52810-52819 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 52820-52829 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 52830-52839 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 52840-52849 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 52850-52859 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 52860-52869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 52870-52879 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 52880-52889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52890-52899 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 52900-52909 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 52910-52919 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 52920-52929 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 52930-52939 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 52940-52949 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 52950-52959 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 52960-52969 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 52970-52979 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 52980-52989 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 52990-52999 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 53000-53009 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 53010-53019 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 53020-53029 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53030-53039 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 53040-53049 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 53050-53059 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 53060-53069 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 53070-53079 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 53080-53089 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 53090-53099 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53100-53109 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 53110-53119 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 53120-53129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53130-53139 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 53140-53149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53150-53159 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53160-53169 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 53170-53179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 53180-53189 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 53190-53199 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 53200-53209 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 53210-53219 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53220-53229 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 28, // 53230-53239 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 53240-53249 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53250-53259 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 53260-53269 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 53270-53279 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 53280-53289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 53290-53299 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 53300-53309 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53310-53319 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 53320-53329 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 53330-53339 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53340-53349 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 53350-53359 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53360-53369 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 53370-53379 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 53380-53389 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53390-53399 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 53400-53409 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 53410-53419 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53420-53429 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 53430-53439 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53440-53449 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 53450-53459 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 53460-53469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 53470-53479 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 53480-53489 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53490-53499 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 53500-53509 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53510-53519 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 53520-53529 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 53530-53539 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 53540-53549 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 53550-53559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 53560-53569 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 53570-53579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53580-53589 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 53590-53599 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 53600-53609 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 53610-53619 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 53620-53629 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 53630-53639 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53640-53649 + 3, 2, 1, 4, 3, 2, 1, 24, 23, 22, // 53650-53659 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 53660-53669 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53670-53679 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53680-53689 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 53690-53699 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53700-53709 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 53710-53719 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53720-53729 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 53730-53739 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 53740-53749 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 53750-53759 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53760-53769 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 53770-53779 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 53780-53789 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 53790-53799 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 53800-53809 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 53810-53819 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53820-53829 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 53830-53839 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 53840-53849 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 53850-53859 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 53860-53869 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53870-53879 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 53880-53889 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 53890-53899 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53900-53909 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 53910-53919 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 53920-53929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 53930-53939 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 53940-53949 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 28, // 53950-53959 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 53960-53969 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 53970-53979 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 53980-53989 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 53990-53999 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54000-54009 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 54010-54019 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54020-54029 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 54030-54039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 54040-54049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 54050-54059 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 54060-54069 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 54070-54079 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 54080-54089 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54090-54099 + 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 54100-54109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54110-54119 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 54120-54129 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 54130-54139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54140-54149 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 54150-54159 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 54160-54169 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54170-54179 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 54180-54189 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 54190-54199 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54200-54209 + 7, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 54210-54219 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 54220-54229 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 54230-54239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54240-54249 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 54250-54259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 54260-54269 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 54270-54279 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 54280-54289 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 54290-54299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54300-54309 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 54310-54319 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 54320-54329 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54330-54339 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 54340-54349 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54350-54359 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 54360-54369 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 54370-54379 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 54380-54389 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54390-54399 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 54400-54409 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 54410-54419 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54420-54429 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 54430-54439 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 54440-54449 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 54450-54459 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 54460-54469 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 54470-54479 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 54480-54489 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 54490-54499 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 54500-54509 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 54510-54519 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 54520-54529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 54530-54539 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 54540-54549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 54550-54559 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 54560-54569 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 54570-54579 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 54580-54589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54590-54599 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54600-54609 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 54610-54619 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 54620-54629 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54630-54639 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 54640-54649 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54650-54659 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 54660-54669 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 30, // 54670-54679 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 54680-54689 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 54690-54699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 54700-54709 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 54710-54719 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 54720-54729 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 54730-54739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54740-54749 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54750-54759 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 54760-54769 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 54770-54779 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 54780-54789 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 54790-54799 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 54800-54809 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 54810-54819 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 54820-54829 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 54830-54839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54840-54849 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 54850-54859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 54860-54869 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 54870-54879 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 54880-54889 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 54890-54899 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 54900-54909 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 54910-54919 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 54920-54929 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54930-54939 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 54940-54949 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 54950-54959 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 54960-54969 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 54970-54979 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 54980-54989 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 54990-54999 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 55000-55009 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55010-55019 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 55020-55029 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 55030-55039 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 55040-55049 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 55050-55059 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55060-55069 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 24, // 55070-55079 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 55080-55089 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55090-55099 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 55100-55109 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 55110-55119 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 55120-55129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 55130-55139 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 55140-55149 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55150-55159 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 55160-55169 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 55170-55179 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 55180-55189 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55190-55199 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 55200-55209 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 55210-55219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 55220-55229 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55230-55239 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 55240-55249 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 32, // 55250-55259 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 55260-55269 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 55270-55279 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55280-55289 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 55290-55299 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55300-55309 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 55310-55319 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55320-55329 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 55330-55339 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 55340-55349 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 55350-55359 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55360-55369 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 55370-55379 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 55380-55389 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 55390-55399 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55400-55409 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 55410-55419 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 55420-55429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 55430-55439 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 55440-55449 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 55450-55459 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 55460-55469 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 55470-55479 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 55480-55489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55490-55499 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55500-55509 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 55510-55519 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 55520-55529 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55530-55539 + 1, 6, 5, 4, 3, 2, 1, 32, 31, 30, // 55540-55549 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 55550-55559 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 55560-55569 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 55570-55579 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 55580-55589 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55590-55599 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 55600-55609 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 55610-55619 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55620-55629 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 55630-55639 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 55640-55649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55650-55659 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 55660-55669 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 55670-55679 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55680-55689 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 55690-55699 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55700-55709 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 55710-55719 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55720-55729 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 55730-55739 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 55740-55749 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 55750-55759 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 55760-55769 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 55770-55779 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 55780-55789 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 55790-55799 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 55800-55809 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 55810-55819 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 55820-55829 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 55830-55839 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 55840-55849 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 55850-55859 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55860-55869 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 55870-55879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 55880-55889 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 55890-55899 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 55900-55909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 55910-55919 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 55920-55929 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 55930-55939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 55940-55949 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 55950-55959 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 55960-55969 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 55970-55979 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 55980-55989 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 55990-55999 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 30, // 56000-56009 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 56010-56019 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 56020-56029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 56030-56039 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 56040-56049 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 56050-56059 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 56060-56069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56070-56079 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 56080-56089 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 56090-56099 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 56100-56109 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 56110-56119 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 56120-56129 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 56130-56139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 56140-56149 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 56150-56159 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 56160-56169 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 56170-56179 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 56180-56189 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 56190-56199 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 28, // 56200-56209 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 56210-56219 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 56220-56229 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 56230-56239 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 56240-56249 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 56250-56259 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 30, // 56260-56269 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 56270-56279 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 56280-56289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 56290-56299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56300-56309 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 56310-56319 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 56320-56329 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 56330-56339 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 56340-56349 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 56350-56359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 56360-56369 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 56370-56379 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 56380-56389 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 56390-56399 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 56400-56409 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 56410-56419 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56420-56429 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 56430-56439 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 56440-56449 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 56450-56459 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 56460-56469 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 56470-56479 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 56480-56489 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56490-56499 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 56500-56509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 56510-56519 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 56520-56529 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 56530-56539 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 56540-56549 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 56550-56559 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 56560-56569 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 56570-56579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56580-56589 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 56590-56599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56600-56609 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 56610-56619 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 56620-56629 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 56630-56639 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 56640-56649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 56650-56659 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 56660-56669 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56670-56679 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 56680-56689 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56690-56699 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56700-56709 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 56710-56719 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56720-56729 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 56730-56739 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 56740-56749 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 56750-56759 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 56760-56769 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 56770-56779 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 56780-56789 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 56790-56799 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 56800-56809 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 56810-56819 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 56820-56829 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 56830-56839 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 56840-56849 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 56850-56859 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 56860-56869 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 56870-56879 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56880-56889 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 56890-56899 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 56900-56909 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56910-56919 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 56920-56929 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56930-56939 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 56940-56949 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 56950-56959 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 56960-56969 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 56970-56979 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 56980-56989 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 38, // 56990-56999 + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 57000-57009 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 57010-57019 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57020-57029 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 57030-57039 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 57040-57049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 57050-57059 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 57060-57069 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 57070-57079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 57080-57089 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 57090-57099 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 57100-57109 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 57110-57119 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57120-57129 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 57130-57139 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 57140-57149 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 57150-57159 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 57160-57169 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 57170-57179 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57180-57189 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 57190-57199 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 57200-57209 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57210-57219 + 1, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 57220-57229 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57230-57239 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57240-57249 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 57250-57259 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 57260-57269 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 57270-57279 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 57280-57289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57290-57299 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 57300-57309 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 57310-57319 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 57320-57329 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57330-57339 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 57340-57349 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57350-57359 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 57360-57369 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 57370-57379 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 57380-57389 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 57390-57399 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 57400-57409 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 57410-57419 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 57420-57429 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 57430-57439 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57440-57449 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 57450-57459 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 57460-57469 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57470-57479 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 57480-57489 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 57490-57499 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 57500-57509 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57510-57519 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 28, // 57520-57529 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 57530-57539 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57540-57549 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 57550-57559 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57560-57569 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57570-57579 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 57580-57589 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 57590-57599 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 57600-57609 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 57610-57619 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57620-57629 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 57630-57639 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 57640-57649 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 57650-57659 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 57660-57669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 57670-57679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 57680-57689 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 57690-57699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 57700-57709 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 57710-57719 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 57720-57729 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 57730-57739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57740-57749 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 57750-57759 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 57760-57769 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 57770-57779 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 57780-57789 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 57790-57799 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 20, // 57800-57809 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 57810-57819 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 57820-57829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 57830-57839 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 57840-57849 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 57850-57859 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 57860-57869 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57870-57879 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 57880-57889 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 57890-57899 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 57900-57909 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 57910-57919 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 57920-57929 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 57930-57939 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 57940-57949 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 57950-57959 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 57960-57969 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 57970-57979 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 57980-57989 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 57990-57999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 58000-58009 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 58010-58019 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 58020-58029 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 58030-58039 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 58040-58049 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 58050-58059 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58060-58069 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 58070-58079 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 58080-58089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 58090-58099 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 58100-58109 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 58110-58119 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 58120-58129 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58130-58139 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 58140-58149 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 58150-58159 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 58160-58169 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 58170-58179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 58180-58189 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 58190-58199 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 58200-58209 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 58210-58219 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 58220-58229 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58230-58239 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 58240-58249 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 58250-58259 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58260-58269 + 1, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 58270-58279 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 58280-58289 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 58290-58299 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 58300-58309 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 58310-58319 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58320-58329 + 7, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 58330-58339 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 58340-58349 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 58350-58359 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 10, // 58360-58369 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 58370-58379 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58380-58389 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 58390-58399 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 58400-58409 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 58410-58419 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 58420-58429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 58430-58439 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58440-58449 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 58450-58459 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58460-58469 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 58470-58479 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 58480-58489 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 58490-58499 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58500-58509 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 58510-58519 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58520-58529 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58530-58539 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 58540-58549 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58550-58559 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58560-58569 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 58570-58579 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 58580-58589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58590-58599 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 58600-58609 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 58610-58619 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58620-58629 + 1, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 58630-58639 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58640-58649 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 58650-58659 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 58660-58669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 58670-58679 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58680-58689 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 58690-58699 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58700-58709 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58710-58719 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58720-58729 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 58730-58739 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58740-58749 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58750-58759 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 58760-58769 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58770-58779 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 42, // 58780-58789 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 58790-58799 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 58800-58809 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 58810-58819 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58820-58829 + 1, 58, 57, 56, 55, 54, 53, 52, 51, 50, // 58830-58839 + 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, // 58840-58849 + 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 58850-58859 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 58860-58869 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 58870-58879 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 58880-58889 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 58890-58899 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 58900-58909 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 58910-58919 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 58920-58929 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 58930-58939 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 58940-58949 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 58950-58959 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 58960-58969 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 58970-58979 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 58980-58989 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 58990-58999 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 59000-59009 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59010-59019 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 59020-59029 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 59030-59039 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59040-59049 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 59050-59059 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 59060-59069 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 59070-59079 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 59080-59089 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 59090-59099 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 59100-59109 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 59110-59119 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 59120-59129 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59130-59139 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 59140-59149 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 59150-59159 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 59160-59169 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59170-59179 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 59180-59189 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 59190-59199 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 59200-59209 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 59210-59219 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59220-59229 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 59230-59239 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 59240-59249 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59250-59259 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 59260-59269 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 59270-59279 + 1, 52, 51, 50, 49, 48, 47, 46, 45, 44, // 59280-59289 + 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, // 59290-59299 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 59300-59309 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 59310-59319 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59320-59329 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 59330-59339 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59340-59349 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 59350-59359 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 59360-59369 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 59370-59379 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 59380-59389 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 59390-59399 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 59400-59409 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 59410-59419 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 59420-59429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59430-59439 + 1, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 59440-59449 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 59450-59459 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 59460-59469 + 1, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 59470-59479 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 59480-59489 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 59490-59499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 59500-59509 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 59510-59519 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 59520-59529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 59530-59539 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 59540-59549 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 59550-59559 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 59560-59569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59570-59579 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 59580-59589 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 59590-59599 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59600-59609 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 59610-59619 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 22, // 59620-59629 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 59630-59639 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59640-59649 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 59650-59659 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 59660-59669 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 59670-59679 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59680-59689 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 59690-59699 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 59700-59709 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59710-59719 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 59720-59729 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59730-59739 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 59740-59749 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 59750-59759 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59760-59769 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 59770-59779 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59780-59789 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 59790-59799 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 59800-59809 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 59810-59819 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59820-59829 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 59830-59839 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 59840-59849 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 59850-59859 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 59860-59869 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 59870-59879 + 7, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 59880-59889 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 59890-59899 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 59900-59909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59910-59919 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 59920-59929 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 59930-59939 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59940-59949 + 1, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 59950-59959 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59960-59969 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 59970-59979 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 59980-59989 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 59990-59999 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60000-60009 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 60010-60019 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 60020-60029 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 60030-60039 + 1, 36, 35, 34, 33, 32, 31, 30, 29, 28, // 60040-60049 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 60050-60059 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 60060-60069 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 60070-60079 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 60080-60089 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60090-60099 + 1, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 60100-60109 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 60110-60119 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 60120-60129 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 60130-60139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 60140-60149 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60150-60159 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 40, // 60160-60169 + 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 60170-60179 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 60180-60189 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 60190-60199 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 60200-60209 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 60210-60219 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 60220-60229 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 60230-60239 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60240-60249 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 60250-60259 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60260-60269 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 60270-60279 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 60280-60289 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 60290-60299 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 60300-60309 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 60310-60319 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60320-60329 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 60330-60339 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 60340-60349 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 60350-60359 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60360-60369 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 60370-60379 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 60380-60389 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 60390-60399 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60400-60409 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 60410-60419 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 60420-60429 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60430-60439 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 60440-60449 + 7, 6, 5, 4, 3, 2, 1, 36, 35, 34, // 60450-60459 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 60460-60469 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 60470-60479 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60480-60489 + 3, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 60490-60499 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 60500-60509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60510-60519 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 60520-60529 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 50, // 60530-60539 + 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, // 60540-60549 + 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 60550-60559 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 60560-60569 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 60570-60579 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 60580-60589 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60590-60599 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 60600-60609 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 60610-60619 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 60620-60629 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 60630-60639 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 60640-60649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 60650-60659 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 60660-60669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 60670-60679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 60680-60689 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60690-60699 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 60700-60709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 60710-60719 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 60720-60729 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 60730-60739 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 60740-60749 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 60750-60759 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 60760-60769 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 60770-60779 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60780-60789 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 60790-60799 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60800-60809 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60810-60819 + 1, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 60820-60829 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 60830-60839 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 60840-60849 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 60850-60859 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 60860-60869 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 60870-60879 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 60880-60889 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 60890-60899 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 60900-60909 + 3, 2, 1, 4, 3, 2, 1, 2, 1, 4, // 60910-60919 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 60920-60929 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 60930-60939 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 60940-60949 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 60950-60959 + 1, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 60960-60969 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 60970-60979 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 60980-60989 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 60990-60999 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 61000-61009 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 61010-61019 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 61020-61029 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61030-61039 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 61040-61049 + 1, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 61050-61059 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 61060-61069 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 61070-61079 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61080-61089 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 22, // 61090-61099 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 61100-61109 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61110-61119 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 61120-61129 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61130-61139 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61140-61149 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 61150-61159 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 42, // 61160-61169 + 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, // 61170-61179 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 61180-61189 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 61190-61199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61200-61209 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61210-61219 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 61220-61229 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 61230-61239 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61240-61249 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 61250-61259 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 61260-61269 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61270-61279 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 61280-61289 + 1, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 61290-61299 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 61300-61309 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 61310-61319 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61320-61329 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 61330-61339 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 61340-61349 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 61350-61359 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 61360-61369 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 61370-61379 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 61380-61389 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61390-61399 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 8, // 61400-61409 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 61410-61419 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 61420-61429 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61430-61439 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 61440-61449 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61450-61459 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 61460-61469 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61470-61479 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 61480-61489 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 61490-61499 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 61500-61509 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 24, // 61510-61519 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 61520-61529 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61530-61539 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 61540-61549 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 2, // 61550-61559 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 61560-61569 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61570-61579 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 61580-61589 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61590-61599 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 61600-61609 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 61610-61619 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 61620-61629 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 61630-61639 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 61640-61649 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 61650-61659 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 61660-61669 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 61670-61679 + 1, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 61680-61689 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61690-61699 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 61700-61709 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 61710-61719 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 61720-61729 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 61730-61739 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61740-61749 + 1, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 61750-61759 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 61760-61769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61770-61779 + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 61780-61789 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 61790-61799 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61800-61809 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 61810-61819 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 61820-61829 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 61830-61839 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 61840-61849 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61850-61859 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61860-61869 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 61870-61879 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 61880-61889 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 61890-61899 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 61900-61909 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 61910-61919 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 61920-61929 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 61930-61939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 61940-61949 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 61950-61959 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 61960-61969 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 61970-61979 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 61980-61989 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 61990-61999 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 62000-62009 + 1, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 62010-62019 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 62020-62029 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 62030-62039 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 62040-62049 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 62050-62059 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62060-62069 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62070-62079 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 62080-62089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 62090-62099 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 62100-62109 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 62110-62119 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 62120-62129 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 62130-62139 + 1, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 62140-62149 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 62150-62159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62160-62169 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 62170-62179 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 62180-62189 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62190-62199 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 62200-62209 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 62210-62219 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62220-62229 + 3, 2, 1, 40, 39, 38, 37, 36, 35, 34, // 62230-62239 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 62240-62249 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 62250-62259 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62260-62269 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 62270-62279 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 62280-62289 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 62290-62299 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 62300-62309 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62310-62319 + 3, 2, 1, 4, 3, 2, 1, 20, 19, 18, // 62320-62329 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 62330-62339 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 62340-62349 + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 62350-62359 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 62360-62369 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62370-62379 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 62380-62389 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62390-62399 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 62400-62409 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 62410-62419 + 3, 2, 1, 36, 35, 34, 33, 32, 31, 30, // 62420-62429 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 62430-62439 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 62440-62449 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 62450-62459 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 62460-62469 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 62470-62479 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 62480-62489 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 62490-62499 + 1, 6, 5, 4, 3, 2, 1, 26, 25, 24, // 62500-62509 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 62510-62519 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62520-62529 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 62530-62539 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 62540-62549 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62550-62559 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 62560-62569 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62570-62579 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62580-62589 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 62590-62599 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 62600-62609 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 62610-62619 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 62620-62629 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 62630-62639 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62640-62649 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 24, // 62650-62659 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 62660-62669 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62670-62679 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 62680-62689 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62690-62699 + 1, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 62700-62709 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62710-62719 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 62720-62729 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62730-62739 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 62740-62749 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 62750-62759 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 62760-62769 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 62770-62779 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62780-62789 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62790-62799 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 62800-62809 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 62810-62819 + 7, 6, 5, 4, 3, 2, 1, 24, 23, 22, // 62820-62829 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 62830-62839 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62840-62849 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62850-62859 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 62860-62869 + 3, 2, 1, 24, 23, 22, 21, 20, 19, 18, // 62870-62879 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 62880-62889 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 62890-62899 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 62900-62909 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62910-62919 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 62920-62929 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 62930-62939 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 62940-62949 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 62950-62959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 62960-62969 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 62970-62979 + 1, 2, 1, 4, 3, 2, 1, 2, 1, 40, // 62980-62989 + 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, // 62990-62999 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 63000-63009 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63010-63019 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 63020-63029 + 1, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 63030-63039 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63040-63049 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 63050-63059 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63060-63069 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 18, // 63070-63079 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 63080-63089 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63090-63099 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 63100-63109 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 63110-63119 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 63120-63129 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63130-63139 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 30, // 63140-63149 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 63150-63159 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63160-63169 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 63170-63179 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 63180-63189 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 63190-63199 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 63200-63209 + 1, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 63210-63219 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 63220-63229 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 63230-63239 + 1, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 63240-63249 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 63250-63259 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 63260-63269 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 63270-63279 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63280-63289 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 12, // 63290-63299 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 63300-63309 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 63310-63319 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 63320-63329 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 63330-63339 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63340-63349 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 63350-63359 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 63360-63369 + 7, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 63370-63379 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 63380-63389 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 63390-63399 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 63400-63409 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 63410-63419 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63420-63429 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 63430-63439 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 63440-63449 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 63450-63459 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 63460-63469 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 63470-63479 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63480-63489 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 22, // 63490-63499 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 63500-63509 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 63510-63519 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63520-63529 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 63530-63539 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63540-63549 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 63550-63559 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 63560-63569 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 63570-63579 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 63580-63589 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 63590-63599 + 1, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 63600-63609 + 1, 6, 5, 4, 3, 2, 1, 12, 11, 10, // 63610-63619 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 63620-63629 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 63630-63639 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 10, // 63640-63649 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 63650-63659 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 63660-63669 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63670-63679 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 63680-63689 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63690-63699 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 63700-63709 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 63710-63719 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 63720-63729 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63730-63739 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 63740-63749 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 63750-63759 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 63760-63769 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 63770-63779 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 63780-63789 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 63790-63799 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 63800-63809 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 63810-63819 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 63820-63829 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 63830-63839 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 63840-63849 + 3, 2, 1, 4, 3, 2, 1, 6, 5, 4, // 63850-63859 + 3, 2, 1, 38, 37, 36, 35, 34, 33, 32, // 63860-63869 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 63870-63879 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 63880-63889 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 63890-63899 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 63900-63909 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 63910-63919 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, // 63920-63929 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 63930-63939 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 28, // 63940-63949 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 63950-63959 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 63960-63969 + 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 63970-63979 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 63980-63989 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 63990-63999 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 64000-64009 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 14, // 64010-64019 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64020-64029 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 64030-64039 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 64040-64049 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64050-64059 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 64060-64069 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64070-64079 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64080-64089 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 64090-64099 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 64100-64109 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64110-64119 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 64120-64129 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 64130-64139 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64140-64149 + 1, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 64150-64159 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64160-64169 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 64170-64179 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 28, // 64180-64189 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 64190-64199 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 64200-64209 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 64210-64219 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 64220-64229 + 1, 6, 5, 4, 3, 2, 1, 34, 33, 32, // 64230-64239 + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, // 64240-64249 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 64250-64259 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64260-64269 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 64270-64279 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 64280-64289 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64290-64299 + 1, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 64300-64309 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 64310-64319 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 64320-64329 + 3, 2, 1, 40, 39, 38, 37, 36, 35, 34, // 64330-64339 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 64340-64349 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 64350-64359 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64360-64369 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 64370-64379 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 64380-64389 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 64390-64399 + 3, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 64400-64409 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 64410-64419 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64420-64429 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 64430-64439 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64440-64449 + 1, 2, 1, 30, 29, 28, 27, 26, 25, 24, // 64450-64459 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 64460-64469 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64470-64479 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 10, // 64480-64489 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 64490-64499 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64500-64509 + 3, 2, 1, 40, 39, 38, 37, 36, 35, 34, // 64510-64519 + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, // 64520-64529 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 64530-64539 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64540-64549 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 64550-64559 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 64560-64569 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 64570-64579 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64580-64589 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64590-64599 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 64600-64609 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 64610-64619 + 1, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 64620-64629 + 3, 2, 1, 28, 27, 26, 25, 24, 23, 22, // 64630-64639 + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, // 64640-64649 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64650-64659 + 1, 2, 1, 4, 3, 2, 1, 12, 11, 10, // 64660-64669 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 64670-64679 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64680-64689 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 64690-64699 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 8, // 64700-64709 + 7, 6, 5, 4, 3, 2, 1, 30, 29, 28, // 64710-64719 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 64720-64729 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 64730-64739 + 7, 6, 5, 4, 3, 2, 1, 16, 15, 14, // 64740-64749 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 64750-64759 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 64760-64769 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64770-64779 + 1, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 64780-64789 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 64790-64799 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64800-64809 + 1, 6, 5, 4, 3, 2, 1, 32, 31, 30, // 64810-64819 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 64820-64829 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 64830-64839 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 64840-64849 + 3, 2, 1, 18, 17, 16, 15, 14, 13, 12, // 64850-64859 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64860-64869 + 1, 6, 5, 4, 3, 2, 1, 2, 1, 12, // 64870-64879 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64880-64889 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64890-64899 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 64900-64909 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 64910-64919 + 1, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 64920-64929 + 7, 6, 5, 4, 3, 2, 1, 14, 13, 12, // 64930-64939 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 64940-64949 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 64950-64959 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 28, // 64960-64969 + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, // 64970-64979 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 64980-64989 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 64990-64999 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 65000-65009 + 1, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 65010-65019 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 4, // 65020-65029 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 65030-65039 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 65040-65049 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 65050-65059 + 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, // 65060-65069 + 1, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 65070-65079 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 10, // 65080-65089 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 65090-65099 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 65100-65109 + 1, 8, 7, 6, 5, 4, 3, 2, 1, 4, // 65110-65119 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 12, // 65120-65129 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 65130-65139 + 1, 6, 5, 4, 3, 2, 1, 20, 19, 18, // 65140-65149 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 65150-65159 + 7, 6, 5, 4, 3, 2, 1, 4, 3, 2, // 65160-65169 + 1, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 65170-65179 + 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, // 65180-65189 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 65190-65199 + 3, 2, 1, 10, 9, 8, 7, 6, 5, 4, // 65200-65209 + 3, 2, 1, 26, 25, 24, 23, 22, 21, 20, // 65210-65219 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 65220-65229 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 65230-65239 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 65240-65249 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 65250-65259 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 18, // 65260-65269 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 65270-65279 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 65280-65289 + 3, 2, 1, 16, 15, 14, 13, 12, 11, 10, // 65290-65299 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 14, // 65300-65309 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 65310-65319 + 3, 2, 1, 4, 3, 2, 1, 26, 25, 24, // 65320-65329 + 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, // 65330-65339 + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 65340-65349 + 3, 2, 1, 4, 3, 2, 1, 14, 13, 12, // 65350-65359 + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 65360-65369 + 1, 10, 9, 8, 7, 6, 5, 4, 3, 2, // 65370-65379 + 1, 12, 11, 10, 9, 8, 7, 6, 5, 4, // 65380-65389 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 65390-65399 + 7, 6, 5, 4, 3, 2, 1, 6, 5, 4, // 65400-65409 + 3, 2, 1, 6, 5, 4, 3, 2, 1, 4, // 65410-65419 + 3, 2, 1, 14, 13, 12, 11, 10, 9, 8, // 65420-65429 + 7, 6, 5, 4, 3, 2, 1, 10, 9, 8, // 65430-65439 + 7, 6, 5, 4, 3, 2, 1, 2, 1, 30, // 65440-65449 + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, // 65450-65459 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 65460-65469 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 18, // 65470-65479 + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, // 65480-65489 + 7, 6, 5, 4, 3, 2, 1, 22, 21, 20, // 65490-65499 + 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, // 65500-65509 + 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, // 65510-65519 + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 65520-65529 + 0, 0, 0, 0, 0, 0, + } + + lohi [256]struct{ lo, hi int } +) + +func init() { + for i, v := range liars { + blk := v >> 24 + x := &lohi[blk] + if x.lo == 0 || i < x.lo { + x.lo = i + } + if i > x.hi { + x.hi = i + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/mathutil/test_deps.go b/Godeps/_workspace/src/github.com/cznic/mathutil/test_deps.go new file mode 100644 index 00000000000..40054dcad2c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/mathutil/test_deps.go @@ -0,0 +1,11 @@ +// Copyright (c) 2014 The mathutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mathutil + +// Pull test dependencies too. +// Enables easy 'go test X' after 'go get X' +import ( +// nothing yet +) diff --git a/Godeps/_workspace/src/github.com/cznic/ql/AUTHORS b/Godeps/_workspace/src/github.com/cznic/ql/AUTHORS new file mode 100644 index 00000000000..0078f5f5b6d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/AUTHORS @@ -0,0 +1,11 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/ql/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/ql/CONTRIBUTORS new file mode 100644 index 00000000000..a96130fd51e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/CONTRIBUTORS @@ -0,0 +1,11 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Dan Kortschak +Jan Mercl <0xjnml@gmail.com> +OpenNota diff --git a/Godeps/_workspace/src/github.com/cznic/ql/LICENSE b/Godeps/_workspace/src/github.com/cznic/ql/LICENSE new file mode 100644 index 00000000000..0d10c02b92e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The ql Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/ql/Makefile b/Godeps/_workspace/src/github.com/cznic/ql/Makefile new file mode 100644 index 00000000000..d4db8a7ddce --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/Makefile @@ -0,0 +1,74 @@ +# Copyright (c) 2014 The ql Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +.PHONY: all clean nuke + +all: editor scanner.go parser.go + go build + go vet || true + golint + go install ./... + make todo + +bench: all + go test -run NONE -bench . + +check: ql.y + goyacc -v /dev/null -o /dev/null $< + +clean: + go clean + rm -f *~ y.go y.tab.c *.out ql.test + +coerce.go: helper.go + if [ -f coerce.go ] ; then rm coerce.go ; fi + go run helper.go | gofmt > $@ + +cover: + t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t + +cpu: ql.test + go test -c + ./$< -test.bench . -test.cpuprofile cpu.out + go tool pprof --lines $< cpu.out + +editor: check scanner.go parser.go coerce.go + go fmt + go test -i + go test + go install + +internalError: + egrep -ho '"internal error.*"' *.go | sort | cat -n + +mem: ql.test + go test -c + ./$< -test.bench . -test.memprofile mem.out + go tool pprof --lines --web --alloc_space $< mem.out + +nuke: + go clean -i + +parser.go: parser.y + goyacc -o $@ $< + sed -i -e 's|//line.*||' -e 's/yyEofCode/yyEOFCode/' $@ + +ql.test: all + +ql.y: doc.go + sed -n '1,/^package/ s/^\/\/ //p' < $< \ + | ebnf2y -o $@ -oe $*.ebnf -start StatementList -pkg $* -p _ + +scanner.go: scanner.l parser.go + golex -o $@ $< + +todo: + @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go *.l parser.y || true + @grep -n TODO *.go *.l parser.y testdata.ql || true + @grep -n BUG *.go *.l parser.y || true + @grep -n println *.go *.l parser.y || true + +later: + @grep -n LATER *.go *.l parser.y || true + @grep -n MAYBE *.go *.l parser.y || true diff --git a/Godeps/_workspace/src/github.com/cznic/ql/README.md b/Godeps/_workspace/src/github.com/cznic/ql/README.md new file mode 100644 index 00000000000..c7a453a32bc --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/README.md @@ -0,0 +1,21 @@ +ql +== + +Package ql is a pure Go embedded (S)QL database. + +Installation + + $ go get github.com/cznic/ql + +Documentation: [godoc.org/github.com/cznic/ql](http://godoc.org/github.com/cznic/ql) + +---- + +Accompanying tool to play with a DB + +Installation + + $ go get github.com/cznic/ql/ql + +Documentation: [godoc.org/github.com/cznic/ql/ql](http://godoc.org/github.com/cznic/ql/ql) + diff --git a/Godeps/_workspace/src/github.com/cznic/ql/all_test.go b/Godeps/_workspace/src/github.com/cznic/ql/all_test.go new file mode 100644 index 00000000000..10308875420 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/all_test.go @@ -0,0 +1,2720 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "crypto/md5" + "database/sql" + "fmt" + "io" + "io/ioutil" + "log" + "math/big" + "math/rand" + "os" + "path" + "path/filepath" + "runtime" + "runtime/debug" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/cznic/strutil" +) + +// Note: All benchmarks report MB/s equal to record/s. +const benchScale = 1e6 + +func init() { + log.SetFlags(log.Flags() | log.Lshortfile) + isTesting = true +} + +func dbg(s string, va ...interface{}) { + if s == "" { + s = strings.Repeat("%v ", len(va)) + } + _, fn, fl, _ := runtime.Caller(1) + fmt.Printf("dbg %s:%d: ", path.Base(fn), fl) + fmt.Printf(s, va...) + fmt.Println() +} + +func caller(s string, va ...interface{}) { + _, fn, fl, _ := runtime.Caller(2) + fmt.Printf("caller: %s:%d: ", path.Base(fn), fl) + fmt.Printf(s, va...) + fmt.Println() + _, fn, fl, _ = runtime.Caller(1) + fmt.Printf("\tcallee: %s:%d: ", path.Base(fn), fl) + fmt.Println() +} + +func use(...interface{}) {} + +func dumpTables3(r *root) { + fmt.Printf("---- r.head %d, r.thead %p\n", r.head, r.thead) + for k, v := range r.tables { + fmt.Printf("%p: %s->%+v\n", v, k, v) + } + fmt.Println("") +} + +func dumpTables2(s storage) { + fmt.Println("****") + h := int64(1) + for h != 0 { + d, err := s.Read(nil, h) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%d: %v\n", h, d) + h = d[0].(int64) + } + fmt.Println("") +} + +func (db *DB) dumpTables() string { + var buf bytes.Buffer + for k, v := range db.root.tables { + buf.WriteString(fmt.Sprintf("%s->%v, %v\n", k, v.h, v.next)) + } + return buf.String() +} + +func fldsString(f []*fld) string { + a := []string{} + for _, v := range f { + a = append(a, v.name) + } + return strings.Join(a, " ") +} + +type testDB interface { + setup() (db *DB, err error) + mark() (err error) + teardown() (err error) +} + +var ( + _ testDB = (*fileTestDB)(nil) + _ testDB = (*memTestDB)(nil) +) + +type memTestDB struct { + db *DB + m0 int64 +} + +func (m *memTestDB) setup() (db *DB, err error) { + if m.db, err = OpenMem(); err != nil { + return + } + + return m.db, nil +} + +func (m *memTestDB) mark() (err error) { + m.m0, err = m.db.store.Verify() + if err != nil { + m.m0 = -1 + } + return +} + +func (m *memTestDB) teardown() (err error) { + if m.m0 < 0 { + return + } + + n, err := m.db.store.Verify() + if err != nil { + return + } + + if g, e := n, m.m0; g != e { + return fmt.Errorf("STORAGE LEAK: allocs: got %d, exp %d", g, e) + } + + return +} + +type fileTestDB struct { + db *DB + gmp0 int + m0 int64 +} + +func (m *fileTestDB) setup() (db *DB, err error) { + m.gmp0 = runtime.GOMAXPROCS(0) + f, err := ioutil.TempFile("", "ql-test-") + if err != nil { + return + } + + if m.db, err = OpenFile(f.Name(), &Options{}); err != nil { + return + } + + return m.db, nil +} + +func (m *fileTestDB) mark() (err error) { + m.m0, err = m.db.store.Verify() + if err != nil { + m.m0 = -1 + } + return +} + +func (m *fileTestDB) teardown() (err error) { + runtime.GOMAXPROCS(m.gmp0) + defer func() { + f := m.db.store.(*file) + errSet(&err, m.db.Close()) + os.Remove(f.f0.Name()) + if f.wal != nil { + os.Remove(f.wal.Name()) + } + }() + + if m.m0 < 0 { + return + } + + n, err := m.db.store.Verify() + if err != nil { + return + } + + if g, e := n, m.m0; g != e { + return fmt.Errorf("STORAGE LEAK: allocs: got %d, exp %d", g, e) + } + return +} + +type osFileTestDB struct { + db *DB + gmp0 int + m0 int64 +} + +func (m *osFileTestDB) setup() (db *DB, err error) { + m.gmp0 = runtime.GOMAXPROCS(0) + f, err := ioutil.TempFile("", "ql-test-osfile") + if err != nil { + return + } + + if m.db, err = OpenFile("", &Options{OSFile: f}); err != nil { + return + } + + return m.db, nil +} + +func (m *osFileTestDB) mark() (err error) { + m.m0, err = m.db.store.Verify() + if err != nil { + m.m0 = -1 + } + return +} + +func (m *osFileTestDB) teardown() (err error) { + runtime.GOMAXPROCS(m.gmp0) + defer func() { + f := m.db.store.(*file) + errSet(&err, m.db.Close()) + os.Remove(f.f0.Name()) + if f.wal != nil { + os.Remove(f.wal.Name()) + } + }() + + if m.m0 < 0 { + return + } + + n, err := m.db.store.Verify() + if err != nil { + return + } + + if g, e := n, m.m0; g != e { + return fmt.Errorf("STORAGE LEAK: allocs: got %d, exp %d", g, e) + } + return +} + +func TestMemStorage(t *testing.T) { + test(t, &memTestDB{}) +} + +func TestFileStorage(t *testing.T) { + test(t, &fileTestDB{}) +} + +func TestOSFileStorage(t *testing.T) { + test(t, &osFileTestDB{}) +} + +var ( + compiledCommit = MustCompile("COMMIT;") + compiledCreate = MustCompile("BEGIN TRANSACTION; CREATE TABLE t (i16 int16, s16 string, s string);") + compiledCreate2 = MustCompile("BEGIN TRANSACTION; CREATE TABLE t (i16 int16, s16 string, s string); COMMIT;") + compiledIns = MustCompile("INSERT INTO t VALUES($1, $2, $3);") + compiledSelect = MustCompile("SELECT * FROM t;") + compiledSelectOrderBy = MustCompile("SELECT * FROM t ORDER BY i16, s16;") + compiledTrunc = MustCompile("BEGIN TRANSACTION; TRUNCATE TABLE t; COMMIT;") +) + +func rnds16(rng *rand.Rand, n int) string { + a := make([]string, n) + for i := range a { + a[i] = fmt.Sprintf("%016x", rng.Int63()) + } + return strings.Join(a, "") +} + +var ( + benchmarkScaleOnce sync.Once + benchmarkSelectOnce = map[string]sync.Once{} +) + +func benchProlog(b *testing.B) { + benchmarkScaleOnce.Do(func() { + b.Logf(` +============================================================= +NOTE: All benchmarks report records/s as %d bytes/s. +=============================================================`, int64(benchScale)) + }) +} + +func benchmarkSelect(b *testing.B, n int, sel List, ts testDB) { + if testing.Verbose() { + benchProlog(b) + id := fmt.Sprintf("%T|%d", ts, n) + once := benchmarkSelectOnce[id] + once.Do(func() { + b.Logf(`Having a table of %d records, each of size 1kB, measure the performance of +%s +`, n, sel) + }) + benchmarkSelectOnce[id] = once + } + + db, err := ts.setup() + if err != nil { + b.Error(err) + return + } + + defer ts.teardown() + + ctx := NewRWCtx() + if _, i, err := db.Execute(ctx, compiledCreate); err != nil { + b.Error(i, err) + return + } + + rng := rand.New(rand.NewSource(42)) + for i := 0; i < n; i++ { + if _, _, err := db.Execute(ctx, compiledIns, int16(rng.Int()), rnds16(rng, 1), rnds16(rng, 63)); err != nil { + b.Error(err) + return + } + } + + if _, i, err := db.Execute(ctx, compiledCommit); err != nil { + b.Error(i, err) + return + } + + b.SetBytes(int64(n) * benchScale) + runtime.GC() + b.ResetTimer() + for i := 0; i < b.N; i++ { + rs, index, err := db.Execute(nil, sel) + if err != nil { + b.Error(index, err) + return + } + + if err = rs[0].Do(false, func(record []interface{}) (bool, error) { return true, nil }); err != nil { + b.Errorf("%v %T(%#v)", err, err, err) + return + } + } + b.StopTimer() + +} + +func BenchmarkSelectMem1kBx1e2(b *testing.B) { + benchmarkSelect(b, 1e2, compiledSelect, &memTestDB{}) +} + +func BenchmarkSelectMem1kBx1e3(b *testing.B) { + benchmarkSelect(b, 1e3, compiledSelect, &memTestDB{}) +} + +func BenchmarkSelectMem1kBx1e4(b *testing.B) { + benchmarkSelect(b, 1e4, compiledSelect, &memTestDB{}) +} + +func BenchmarkSelectMem1kBx1e5(b *testing.B) { + benchmarkSelect(b, 1e5, compiledSelect, &memTestDB{}) +} + +func BenchmarkSelectFile1kBx1e2(b *testing.B) { + benchmarkSelect(b, 1e2, compiledSelect, &fileTestDB{}) +} + +func BenchmarkSelectFile1kBx1e3(b *testing.B) { + benchmarkSelect(b, 1e3, compiledSelect, &fileTestDB{}) +} + +func BenchmarkSelectFile1kBx1e4(b *testing.B) { + benchmarkSelect(b, 1e4, compiledSelect, &fileTestDB{}) +} + +func BenchmarkSelectFile1kBx1e5(b *testing.B) { + benchmarkSelect(b, 1e5, compiledSelect, &fileTestDB{}) +} + +func BenchmarkSelectOrderedMem1kBx1e2(b *testing.B) { + benchmarkSelect(b, 1e2, compiledSelectOrderBy, &memTestDB{}) +} + +func BenchmarkSelectOrderedMem1kBx1e3(b *testing.B) { + benchmarkSelect(b, 1e3, compiledSelectOrderBy, &memTestDB{}) +} + +func BenchmarkSelectOrderedMem1kBx1e4(b *testing.B) { + benchmarkSelect(b, 1e4, compiledSelectOrderBy, &memTestDB{}) +} + +func BenchmarkSelectOrderedFile1kBx1e2(b *testing.B) { + benchmarkSelect(b, 1e2, compiledSelectOrderBy, &fileTestDB{}) +} + +func BenchmarkSelectOrderedFile1kBx1e3(b *testing.B) { + benchmarkSelect(b, 1e3, compiledSelectOrderBy, &fileTestDB{}) +} + +func BenchmarkSelectOrderedFile1kBx1e4(b *testing.B) { + benchmarkSelect(b, 1e4, compiledSelectOrderBy, &fileTestDB{}) +} + +func TestString(t *testing.T) { + for _, v := range testdata { + a := strings.Split(v, "\n|") + v = a[0] + v = strings.Replace(v, "∨", "|", -1) + v = strings.Replace(v, "⩖", "||", -1) + l, err := Compile(v) + if err != nil { + continue + } + + if s := l.String(); len(s) == 0 { + t.Fatal("List.String() returned empty string") + } + } +} + +var benchmarkInsertOnce = map[string]sync.Once{} + +func benchmarkInsert(b *testing.B, batch, total int, ts testDB) { + if testing.Verbose() { + benchProlog(b) + id := fmt.Sprintf("%T|%d|%d", ts, batch, total) + once := benchmarkInsertOnce[id] + once.Do(func() { + b.Logf(`In batches of %d record(s), insert a total of %d records, each of size 1kB, into a table. + +`, batch, total) + }) + benchmarkInsertOnce[id] = once + } + + if total%batch != 0 { + b.Fatal("internal error 001") + } + + db, err := ts.setup() + if err != nil { + b.Error(err) + return + } + + defer ts.teardown() + + ctx := NewRWCtx() + if _, i, err := db.Execute(ctx, compiledCreate2); err != nil { + b.Error(i, err) + return + } + + s := fmt.Sprintf(`(0, "0123456789abcdef", "%s"),`, rnds16(rand.New(rand.NewSource(42)), 63)) + var buf bytes.Buffer + buf.WriteString("BEGIN TRANSACTION; INSERT INTO t VALUES\n") + for i := 0; i < batch; i++ { + buf.WriteString(s) + } + buf.WriteString("; COMMIT;") + ins, err := Compile(buf.String()) + if err != nil { + b.Error(err) + return + } + + b.SetBytes(int64(total) * benchScale) + runtime.GC() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for n := 0; n != total; n += batch { + if _, _, err = db.Execute(ctx, ins); err != nil { + b.Error(err) + return + } + } + b.StopTimer() + if _, _, err = db.Execute(ctx, compiledTrunc); err != nil { + b.Error(err) + } + b.StartTimer() + } + b.StopTimer() +} + +func BenchmarkInsertMem1kBn1e0t1e2(b *testing.B) { + benchmarkInsert(b, 1e0, 1e2, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e0t1e2(b *testing.B) { + benchmarkInsert(b, 1e0, 1e2, &fileTestDB{}) +} + +//============================================================================= + +func BenchmarkInsertMem1kBn1e1t1e2(b *testing.B) { + benchmarkInsert(b, 1e1, 1e2, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e1t1e2(b *testing.B) { + benchmarkInsert(b, 1e1, 1e2, &fileTestDB{}) +} + +func BenchmarkInsertMem1kBn1e1t1e3(b *testing.B) { + benchmarkInsert(b, 1e1, 1e3, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e1t1e3(b *testing.B) { + benchmarkInsert(b, 1e1, 1e3, &fileTestDB{}) +} + +//============================================================================= + +func BenchmarkInsertMem1kBn1e2t1e2(b *testing.B) { + benchmarkInsert(b, 1e2, 1e2, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e2t1e2(b *testing.B) { + benchmarkInsert(b, 1e2, 1e2, &fileTestDB{}) +} + +func BenchmarkInsertMem1kBn1e2t1e3(b *testing.B) { + benchmarkInsert(b, 1e2, 1e3, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e2t1e3(b *testing.B) { + benchmarkInsert(b, 1e2, 1e3, &fileTestDB{}) +} + +func BenchmarkInsertMem1kBn1e2t1e4(b *testing.B) { + benchmarkInsert(b, 1e2, 1e4, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e2t1e4(b *testing.B) { + benchmarkInsert(b, 1e2, 1e4, &fileTestDB{}) +} + +//============================================================================= + +func BenchmarkInsertMem1kBn1e3t1e3(b *testing.B) { + benchmarkInsert(b, 1e3, 1e3, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e3t1e3(b *testing.B) { + benchmarkInsert(b, 1e3, 1e3, &fileTestDB{}) +} + +func BenchmarkInsertMem1kBn1e3t1e4(b *testing.B) { + benchmarkInsert(b, 1e3, 1e4, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e3t1e4(b *testing.B) { + benchmarkInsert(b, 1e3, 1e4, &fileTestDB{}) +} + +func BenchmarkInsertMem1kBn1e3t1e5(b *testing.B) { + benchmarkInsert(b, 1e3, 1e5, &memTestDB{}) +} + +func BenchmarkInsertFile1kBn1e3t1e5(b *testing.B) { + benchmarkInsert(b, 1e3, 1e5, &fileTestDB{}) +} + +func TestReopen(t *testing.T) { + f, err := ioutil.TempFile("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + nm := f.Name() + if err = f.Close(); err != nil { + t.Fatal(err) + } + + defer func() { + if nm != "" { + os.Remove(nm) + } + }() + + db, err := OpenFile(nm, &Options{}) + if err != nil { + t.Error(err) + return + } + + for _, tn := range "abc" { + ql := fmt.Sprintf(` +BEGIN TRANSACTION; + CREATE TABLE %c (i int, s string); + INSERT INTO %c VALUES (%d, "<%c>"); +COMMIT; + `, tn, tn, tn-'a'+42, tn) + _, i, err := db.Run(NewRWCtx(), ql) + if err != nil { + db.Close() + t.Error(i, err) + return + } + } + + if err = db.Close(); err != nil { + t.Error(err) + return + } + + db, err = OpenFile(nm, &Options{}) + if err != nil { + t.Error(err) + return + } + + defer func() { + if err = db.Close(); err != nil { + t.Error(err) + } + }() + + if _, _, err = db.Run(NewRWCtx(), "BEGIN TRANSACTION; DROP TABLE b; COMMIT;"); err != nil { + t.Error(err) + return + } + + for _, tn := range "ac" { + ql := fmt.Sprintf("SELECT * FROM %c;", tn) + rs, i, err := db.Run(NewRWCtx(), ql) + if err != nil { + t.Error(i, err) + return + } + + if rs == nil { + t.Error(rs) + return + } + + rid := 0 + if err = rs[0].Do(false, func(r []interface{}) (bool, error) { + rid++ + if rid != 1 { + return false, fmt.Errorf("rid %d", rid) + } + + if g, e := recStr(r), fmt.Sprintf(`%d, "<%c>"`, tn-'a'+42, tn); g != e { + return false, fmt.Errorf("g `%s`, e `%s`", g, e) + } + + return true, nil + }); err != nil { + t.Error(err) + return + } + } +} + +func recStr(data []interface{}) string { + a := make([]string, len(data)) + for i, v := range data { + switch x := v.(type) { + case string: + a[i] = fmt.Sprintf("%q", x) + default: + a[i] = fmt.Sprint(x) + } + } + return strings.Join(a, ", ") +} + +//TODO +test long blob types with multiple inner chunks. + +func TestLastInsertID(t *testing.T) { + table := []struct { + ql string + id int + }{ + // 0 + {`BEGIN TRANSACTION; CREATE TABLE t (c int); COMMIT`, 0}, + {`BEGIN TRANSACTION; INSERT INTO t VALUES (41); COMMIT`, 1}, + {`BEGIN TRANSACTION; INSERT INTO t VALUES (42);`, 2}, + {`INSERT INTO t VALUES (43)`, 3}, + {`ROLLBACK; BEGIN TRANSACTION; INSERT INTO t VALUES (44); COMMIT`, 4}, + + //5 + {`BEGIN TRANSACTION; INSERT INTO t VALUES (45); COMMIT`, 5}, + {` + BEGIN TRANSACTION; + INSERT INTO t VALUES (46); // 6 + BEGIN TRANSACTION; + INSERT INTO t VALUES (47); // 7 + INSERT INTO t VALUES (48); // 8 + ROLLBACK; + INSERT INTO t VALUES (49); // 9 + COMMIT`, 9}, + {` + BEGIN TRANSACTION; + INSERT INTO t VALUES (50); // 10 + BEGIN TRANSACTION; + INSERT INTO t VALUES (51); // 11 + INSERT INTO t VALUES (52); // 12 + ROLLBACK; + COMMIT;`, 10}, + {`BEGIN TRANSACTION; INSERT INTO t VALUES (53); ROLLBACK`, 10}, + {`BEGIN TRANSACTION; INSERT INTO t VALUES (54); COMMIT`, 14}, + // 10 + {`BEGIN TRANSACTION; CREATE TABLE u (c int); COMMIT`, 14}, + {` + BEGIN TRANSACTION; + INSERT INTO t SELECT * FROM u; + COMMIT;`, 14}, + {`BEGIN TRANSACTION; INSERT INTO u VALUES (150); COMMIT`, 15}, + {` + BEGIN TRANSACTION; + INSERT INTO t SELECT * FROM u; + COMMIT;`, 16}, + } + + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + ctx := NewRWCtx() + for i, test := range table { + l, err := Compile(test.ql) + if err != nil { + t.Fatal(i, err) + } + + if _, _, err = db.Execute(ctx, l); err != nil { + t.Fatal(i, err) + } + + if g, e := ctx.LastInsertID, int64(test.id); g != e { + t.Fatal(i, g, e) + } + } +} + +func ExampleTCtx_lastInsertID() { + ins := MustCompile("BEGIN TRANSACTION; INSERT INTO t VALUES ($1); COMMIT;") + + db, err := OpenMem() + if err != nil { + panic(err) + } + + ctx := NewRWCtx() + if _, _, err = db.Run(ctx, ` + BEGIN TRANSACTION; + CREATE TABLE t (c int); + INSERT INTO t VALUES (1), (2), (3); + COMMIT; + `); err != nil { + panic(err) + } + + if _, _, err = db.Execute(ctx, ins, int64(42)); err != nil { + panic(err) + } + + id := ctx.LastInsertID + rs, _, err := db.Run(ctx, `SELECT * FROM t WHERE id() == $1`, id) + if err != nil { + panic(err) + } + + if err = rs[0].Do(false, func(data []interface{}) (more bool, err error) { + fmt.Println(data) + return true, nil + }); err != nil { + panic(err) + } + // Output: + // [42] +} + +func Example_recordsetFields() { + // See RecordSet.Fields documentation + + db, err := OpenMem() + if err != nil { + panic(err) + } + + ctx := NewRWCtx() + rs, _, err := db.Run(ctx, ` + BEGIN TRANSACTION; + CREATE TABLE t (s string, i int); + CREATE TABLE u (s string, i int); + INSERT INTO t VALUES + ("a", 1), + ("a", 2), + ("b", 3), + ("b", 4), + ; + INSERT INTO u VALUES + ("A", 10), + ("A", 20), + ("B", 30), + ("B", 40), + ; + COMMIT; + + // [0]: Fields are not computable. + SELECT * FROM noTable; + + // [1]: Fields are computable even when Do will fail (table noTable does not exist). + SELECT X AS Y FROM noTable; + + // [2]: Both Fields and Do are okay. + SELECT t.s+u.s as a, t.i+u.i as b, "noName", "name" as Named FROM t, u; + + // [3]: Filds are computable even when Do will fail (uknown column a). + SELECT DISTINCT s as S, sum(i) as I FROM ( + SELECT t.s+u.s as s, t.i+u.i, 3 as i FROM t, u; + ) + GROUP BY a + ORDER BY d; + + // [4]: Fields are computable even when Do will fail on missing $1. + SELECT DISTINCT * FROM ( + SELECT t.s+u.s as S, t.i+u.i, 3 as I FROM t, u; + ) + WHERE I < $1 + ORDER BY S; + ` /* , 42 */) // <-- $1 missing + if err != nil { + panic(err) + } + + for i, v := range rs { + fields, err := v.Fields() + switch { + case err != nil: + fmt.Printf("Fields[%d]: error: %s\n", i, err) + default: + fmt.Printf("Fields[%d]: %#v\n", i, fields) + } + if err = v.Do( + true, + func(data []interface{}) (more bool, err error) { + fmt.Printf(" Do[%d]: %#v\n", i, data) + return false, nil + }, + ); err != nil { + fmt.Printf(" Do[%d]: error: %s\n", i, err) + } + } + // Output: + // Fields[0]: error: table noTable does not exist + // Do[0]: error: table noTable does not exist + // Fields[1]: []string{"Y"} + // Do[1]: error: table noTable does not exist + // Fields[2]: []string{"a", "b", "", "Named"} + // Do[2]: []interface {}{"a", "b", "", "Named"} + // Fields[3]: []string{"S", "I"} + // Do[3]: error: unknown column a + // Fields[4]: []string{"S", "", "I"} + // Do[4]: error: missing $1 +} + +func TestRowsAffected(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + ctx := NewRWCtx() + ctx.LastInsertID, ctx.RowsAffected = -1, -1 + if _, _, err = db.Run(ctx, ` + BEGIN TRANSACTION; + CREATE TABLE t (i int); + COMMIT; + `); err != nil { + t.Fatal(err) + } + + if g, e := ctx.LastInsertID, int64(0); g != e { + t.Fatal(g, e) + } + + if g, e := ctx.RowsAffected, int64(0); g != e { + t.Fatal(g, e) + } + + if _, _, err = db.Run(ctx, ` + BEGIN TRANSACTION; + INSERT INTO t VALUES (1), (2), (3); + COMMIT; + `); err != nil { + t.Fatal(err) + } + + if g, e := ctx.LastInsertID, int64(3); g != e { + t.Fatal(g, e) + } + + if g, e := ctx.RowsAffected, int64(3); g != e { + t.Fatal(g, e) + } + + if _, _, err = db.Run(ctx, ` + BEGIN TRANSACTION; + INSERT INTO t + SELECT * FROM t WHERE i == 2; + COMMIT; + `); err != nil { + t.Fatal(err) + } + + if g, e := ctx.LastInsertID, int64(4); g != e { + t.Fatal(g, e) + } + + if g, e := ctx.RowsAffected, int64(1); g != e { + t.Fatal(g, e) + } + + if _, _, err = db.Run(ctx, ` + BEGIN TRANSACTION; + DELETE FROM t WHERE i == 2; + COMMIT; + `); err != nil { + t.Fatal(err) + } + + if g, e := ctx.LastInsertID, int64(4); g != e { + t.Fatal(g, e) + } + + if g, e := ctx.RowsAffected, int64(2); g != e { + t.Fatal(g, e) + } + + if _, _, err = db.Run(ctx, ` + BEGIN TRANSACTION; + UPDATE t i = 42 WHERE i == 3; + COMMIT; + `); err != nil { + t.Fatal(err) + } + + if g, e := ctx.LastInsertID, int64(4); g != e { + t.Fatal(g, e) + } + + if g, e := ctx.RowsAffected, int64(1); g != e { + t.Fatal(g, e) + } +} + +func dumpDB(db *DB, tag string) (string, error) { + var buf bytes.Buffer + f := strutil.IndentFormatter(&buf, "\t") + f.Format("---- %s%i\n", tag) + for nm, tab := range db.root.tables { + h := tab.head + f.Format("%u%q: head %d, scols0 %q, scols %q%i\n", nm, h, cols2meta(tab.cols0), cols2meta(tab.cols)) + for h != 0 { + rec, err := db.store.Read(nil, h, tab.cols...) + if err != nil { + return "", err + } + + f.Format("record @%d: %v\n", h, rec) + h = rec[0].(int64) + } + f.Format("%u") + for i, v := range tab.indices { + if v == nil { + continue + } + + xname := v.name + cname := "id()" + if i != 0 { + cname = tab.cols0[i-1].name + } + f.Format("index %s on %s%i\n", xname, cname) + it, _, err := v.x.Seek(nil) + if err != nil { + return "", err + } + + for { + k, v, err := it.Next() + if err != nil { + if err == io.EOF { + break + } + + return "", err + } + + f.Format("%v: %v\n", k, v) + } + f.Format("%u") + } + } + + return buf.String(), nil +} + +func testIndices(db *DB, t *testing.T) { + ctx := NewRWCtx() + var err error + e := func(q string) { + if _, _, err = db.Run(ctx, q); err != nil { + t.Fatal(err) + } + + s, err := dumpDB(db, "post\n\t"+q) + if err != nil { + t.Fatal(err) + } + + t.Logf("%s\n\n", s) + if db.isMem { + return + } + + nm := db.Name() + if err = db.Close(); err != nil { + t.Fatal(err) + } + + if db, err = OpenFile(nm, &Options{}); err != nil { + t.Fatal(err) + } + + if s, err = dumpDB(db, "reopened"); err != nil { + t.Fatal(err) + } + + t.Logf("%s\n\n", s) + } + + e(` BEGIN TRANSACTION; + CREATE TABLE t (i int); + COMMIT;`) + e(` BEGIN TRANSACTION; + CREATE INDEX x ON t (id()); + COMMIT;`) + e(` BEGIN TRANSACTION; + INSERT INTO t VALUES(42); + COMMIT;`) + e(` BEGIN TRANSACTION; + INSERT INTO t VALUES(24); + COMMIT;`) + e(` BEGIN TRANSACTION; + CREATE INDEX xi ON t (i); + COMMIT;`) + e(` BEGIN TRANSACTION; + INSERT INTO t VALUES(1); + COMMIT;`) + e(` BEGIN TRANSACTION; + INSERT INTO t VALUES(999); + COMMIT;`) + e(` BEGIN TRANSACTION; + UPDATE t i = 240 WHERE i == 24; + COMMIT;`) + e(` BEGIN TRANSACTION; + DELETE FROM t WHERE i == 240; + COMMIT;`) + e(` BEGIN TRANSACTION; + TRUNCATE TABLE t; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int, s string); + CREATE INDEX xi ON t (i); + INSERT INTO t VALUES (42, "foo"); + COMMIT;`) + e(` BEGIN TRANSACTION; + ALTER TABLE t DROP COLUMN i; + COMMIT;`) + + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + INSERT INTO t SELECT 10*i FROM t; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + COMMIT; + BEGIN TRANSACTION; + INSERT INTO t SELECT 10*i FROM t; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + DROP INDEX x; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + COMMIT; + BEGIN TRANSACTION; + DROP INDEX x; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + COMMIT;`) + e(` + BEGIN TRANSACTION; + DROP INDEX x; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + ALTER TABLE t ADD s string; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + COMMIT;`) + e(` BEGIN TRANSACTION; + ALTER TABLE t ADD s string; + COMMIT;`) + e(` BEGIN TRANSACTION; + DROP TABLE IF EXISTS t; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(42); + COMMIT;`) + + if err = db.Close(); err != nil { + t.Fatal(err) + } +} + +func TestIndices(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + testIndices(db, t) + dir, err := ioutil.TempDir("", "ql-test") + + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + + nm := filepath.Join(dir, "ql.db") + db, err = OpenFile(nm, &Options{CanCreate: true}) + if err != nil { + t.Fatal(err) + } + + testIndices(db, t) +} + +var benchmarkInsertBoolOnce = map[string]sync.Once{} + +func benchmarkInsertBool(b *testing.B, db *DB, size int, selectivity float64, index bool, teardown func()) { + if testing.Verbose() { + benchProlog(b) + id := fmt.Sprintf("%t|%d|%g|%t", db.isMem, size, selectivity, index) + once := benchmarkInsertBoolOnce[id] + once.Do(func() { + s := "INDEXED" + if !index { + s = "NON " + s + } + b.Logf(`Insert %d records into a table having a single bool %s column. Batch size: 1 record. + +`, size, s) + }) + benchmarkInsertBoolOnce[id] = once + } + + if teardown != nil { + defer teardown() + } + + ctx := NewRWCtx() + if _, _, err := db.Run(ctx, ` + BEGIN TRANSACTION; + CREATE TABLE t (b bool); + `); err != nil { + b.Fatal(err) + } + + if index { + if _, _, err := db.Run(ctx, ` + CREATE INDEX x ON t (b); + `); err != nil { + b.Fatal(err) + } + } + + ins, err := Compile("INSERT INTO t VALUES($1);") + if err != nil { + b.Fatal(err) + } + + trunc, err := Compile("TRUNCATE TABLE t;") + if err != nil { + b.Fatal(err) + } + + begin, err := Compile("BEGIN TRANSACTION;") + if err != nil { + b.Fatal(err) + } + + commit, err := Compile("COMMIT;") + if err != nil { + b.Fatal(err) + } + + rng := rand.New(rand.NewSource(42)) + debug.FreeOSMemory() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + if i != 0 { + if _, _, err = db.Execute(ctx, begin); err != nil { + b.Fatal(err) + } + } + + if _, _, err = db.Execute(ctx, trunc); err != nil { + b.Fatal(err) + } + + b.StartTimer() + for j := 0; j < size; j++ { + if _, _, err = db.Execute(ctx, ins, rng.Float64() < selectivity); err != nil { + b.Fatal(err) + } + } + if _, _, err = db.Execute(ctx, commit); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.SetBytes(int64(size) * benchScale) +} + +func benchmarkInsertBoolMem(b *testing.B, size int, sel float64, index bool) { + db, err := OpenMem() + if err != nil { + b.Fatal(err) + } + + benchmarkInsertBool(b, db, size, sel, index, nil) +} + +func BenchmarkInsertBoolMemNoX1e1(b *testing.B) { + benchmarkInsertBoolMem(b, 1e1, 0.5, false) +} + +func BenchmarkInsertBoolMemX1e1(b *testing.B) { + benchmarkInsertBoolMem(b, 1e1, 0.5, true) +} + +func BenchmarkInsertBoolMemNoX1e2(b *testing.B) { + benchmarkInsertBoolMem(b, 1e2, 0.5, false) +} + +func BenchmarkInsertBoolMemX1e2(b *testing.B) { + benchmarkInsertBoolMem(b, 1e2, 0.5, true) +} + +func BenchmarkInsertBoolMemNoX1e3(b *testing.B) { + benchmarkInsertBoolMem(b, 1e3, 0.5, false) +} + +func BenchmarkInsertBoolMemX1e3(b *testing.B) { + benchmarkInsertBoolMem(b, 1e3, 0.5, true) +} + +func BenchmarkInsertBoolMemNoX1e4(b *testing.B) { + benchmarkInsertBoolMem(b, 1e4, 0.5, false) +} + +func BenchmarkInsertBoolMemX1e4(b *testing.B) { + benchmarkInsertBoolMem(b, 1e4, 0.5, true) +} + +func BenchmarkInsertBoolMemNoX1e5(b *testing.B) { + benchmarkInsertBoolMem(b, 1e5, 0.5, false) +} + +func BenchmarkInsertBoolMemX1e5(b *testing.B) { + benchmarkInsertBoolMem(b, 1e5, 0.5, true) +} + +func benchmarkInsertBoolFile(b *testing.B, size int, sel float64, index bool) { + dir, err := ioutil.TempDir("", "ql-bench-") + if err != nil { + b.Fatal(err) + } + + n := runtime.GOMAXPROCS(0) + db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true}) + if err != nil { + b.Fatal(err) + } + + benchmarkInsertBool(b, db, size, sel, index, func() { + runtime.GOMAXPROCS(n) + db.Close() + os.RemoveAll(dir) + }) +} + +func BenchmarkInsertBoolFileNoX1e1(b *testing.B) { + benchmarkInsertBoolFile(b, 1e1, 0.5, false) +} + +func BenchmarkInsertBoolFileX1e1(b *testing.B) { + benchmarkInsertBoolFile(b, 1e1, 0.5, true) +} + +func BenchmarkInsertBoolFileNoX1e2(b *testing.B) { + benchmarkInsertBoolFile(b, 1e2, 0.5, false) +} + +func BenchmarkInsertBoolFileX1e2(b *testing.B) { + benchmarkInsertBoolFile(b, 1e2, 0.5, true) +} + +func BenchmarkInsertBoolFileNoX1e3(b *testing.B) { + benchmarkInsertBoolFile(b, 1e3, 0.5, false) +} + +func BenchmarkInsertBoolFileX1e3(b *testing.B) { + benchmarkInsertBoolFile(b, 1e3, 0.5, true) +} + +var benchmarkSelectBoolOnce = map[string]sync.Once{} + +func benchmarkSelectBool(b *testing.B, db *DB, size int, selectivity float64, index bool, teardown func()) { + sel, err := Compile("SELECT * FROM t WHERE b;") + if err != nil { + b.Fatal(err) + } + + if testing.Verbose() { + benchProlog(b) + id := fmt.Sprintf("%t|%d|%g|%t", db.isMem, size, selectivity, index) + once := benchmarkSelectBoolOnce[id] + once.Do(func() { + s := "INDEXED" + if !index { + s = "NON " + s + } + b.Logf(`A table has a single %s bool column b. Insert %d records with a random bool value, +%.0f%% of them are true. Measure the performance of +%s +`, s, size, 100*selectivity, sel) + }) + benchmarkSelectBoolOnce[id] = once + } + + if teardown != nil { + defer teardown() + } + + ctx := NewRWCtx() + if _, _, err := db.Run(ctx, ` + BEGIN TRANSACTION; + CREATE TABLE t (b bool); + `); err != nil { + b.Fatal(err) + } + + if index { + if _, _, err := db.Run(ctx, ` + CREATE INDEX x ON t (b); + `); err != nil { + b.Fatal(err) + } + } + + ins, err := Compile("INSERT INTO t VALUES($1);") + if err != nil { + b.Fatal(err) + } + + var n int64 + rng := rand.New(rand.NewSource(42)) + for j := 0; j < size; j++ { + v := rng.Float64() < selectivity + if v { + n++ + } + if _, _, err = db.Execute(ctx, ins, v); err != nil { + b.Fatal(err) + } + } + + if _, _, err := db.Run(ctx, "COMMIT;"); err != nil { + b.Fatal(err) + } + + debug.FreeOSMemory() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var m int64 + rss, _, err := db.Execute(nil, sel) + if err != nil { + b.Fatal(err) + } + + if err = rss[0].Do(false, func([]interface{}) (bool, error) { + m++ + return true, nil + }); err != nil { + b.Fatal(err) + } + if g, e := n, m; g != e { + b.Fatal(g, e) + } + } + b.StopTimer() + b.SetBytes(n * benchScale) +} + +func benchmarkSelectBoolMem(b *testing.B, size int, sel float64, index bool) { + db, err := OpenMem() + if err != nil { + b.Fatal(err) + } + + benchmarkSelectBool(b, db, size, sel, index, nil) +} + +// ---- + +func BenchmarkSelectBoolMemNoX1e1Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e1, 0.5, false) +} + +func BenchmarkSelectBoolMemX1e1Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e1, 0.5, true) +} + +func BenchmarkSelectBoolMemNoX1e2Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e2, 0.5, false) +} + +func BenchmarkSelectBoolMemX1e2Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e2, 0.5, true) +} + +func BenchmarkSelectBoolMemNoX1e3Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e3, 0.5, false) +} + +func BenchmarkSelectBoolMemX1e3Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e3, 0.5, true) +} + +func BenchmarkSelectBoolMemNoX1e4Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e4, 0.5, false) +} + +func BenchmarkSelectBoolMemX1e4Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e4, 0.5, true) +} + +func BenchmarkSelectBoolMemNoX1e5Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e5, 0.5, false) +} + +func BenchmarkSelectBoolMemX1e5Perc50(b *testing.B) { + benchmarkSelectBoolMem(b, 1e5, 0.5, true) +} + +// ---- + +func BenchmarkSelectBoolMemNoX1e1Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e1, 0.05, false) +} + +func BenchmarkSelectBoolMemX1e1Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e1, 0.05, true) +} + +func BenchmarkSelectBoolMemNoX1e2Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e2, 0.05, false) +} + +func BenchmarkSelectBoolMemX1e2Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e2, 0.05, true) +} + +func BenchmarkSelectBoolMemNoX1e3Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e3, 0.05, false) +} + +func BenchmarkSelectBoolMemX1e3Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e3, 0.05, true) +} + +func BenchmarkSelectBoolMemNoX1e4Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e4, 0.05, false) +} + +func BenchmarkSelectBoolMemX1e4Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e4, 0.05, true) +} + +func BenchmarkSelectBoolMemNoX1e5Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e5, 0.05, false) +} + +func BenchmarkSelectBoolMemX1e5Perc5(b *testing.B) { + benchmarkSelectBoolMem(b, 1e5, 0.05, true) +} + +func benchmarkSelectBoolFile(b *testing.B, size int, sel float64, index bool) { + dir, err := ioutil.TempDir("", "ql-bench-") + if err != nil { + b.Fatal(err) + } + + n := runtime.GOMAXPROCS(0) + db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true}) + if err != nil { + b.Fatal(err) + } + + benchmarkSelectBool(b, db, size, sel, index, func() { + runtime.GOMAXPROCS(n) + db.Close() + os.RemoveAll(dir) + }) +} + +// ---- + +func BenchmarkSelectBoolFileNoX1e1Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.5, false) +} + +func BenchmarkSelectBoolFileX1e1Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.5, true) +} + +func BenchmarkSelectBoolFileNoX1e2Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.5, false) +} + +func BenchmarkSelectBoolFileX1e2Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.5, true) +} + +func BenchmarkSelectBoolFileNoX1e3Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.5, false) +} + +func BenchmarkSelectBoolFileX1e3Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.5, true) +} + +func BenchmarkSelectBoolFileNoX1e4Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.5, false) +} + +func BenchmarkSelectBoolFileX1e4Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.5, true) +} + +// ---- + +func BenchmarkSelectBoolFileNoX1e1Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.05, false) +} + +func BenchmarkSelectBoolFileX1e1Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.05, true) +} + +func BenchmarkSelectBoolFileNoX1e2Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.05, false) +} + +func BenchmarkSelectBoolFileX1e2Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.05, true) +} + +func BenchmarkSelectBoolFileNoX1e3Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.05, false) +} + +func BenchmarkSelectBoolFileX1e3Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.05, true) +} + +func BenchmarkSelectBoolFileNoX1e4Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.05, false) +} + +func BenchmarkSelectBoolFileX1e4Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.05, true) +} + +func TestIndex(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + ctx := NewRWCtx() + if _, _, err := db.Run(ctx, ` + BEGIN TRANSACTION; + CREATE TABLE t (b bool); + `); err != nil { + t.Fatal(err) + } + + if _, _, err := db.Run(ctx, ` + CREATE INDEX x ON t (b); + `); err != nil { + t.Fatal(err) + } + + ins, err := Compile("INSERT INTO t VALUES($1);") + if err != nil { + t.Fatal(err) + } + + size, selectivity := int(1e1), 0.5 + rng := rand.New(rand.NewSource(42)) + var n int64 + for j := 0; j < size; j++ { + v := rng.Float64() < selectivity + if v { + n++ + t.Logf("id %d <- true", j+1) + } + if _, _, err = db.Execute(ctx, ins, v); err != nil { + t.Fatal(err) + } + } + + if _, _, err := db.Run(ctx, "COMMIT;"); err != nil { + t.Fatal(err) + } + + s, err := dumpDB(db, "") + if err != nil { + t.Fatal(err) + } + + t.Logf("n: %d\n%s", n, s) + sel, err := Compile("SELECT id(), b FROM t WHERE b;") + if err != nil { + t.Fatal(err) + } + + var m int64 + rss, _, err := db.Execute(nil, sel) + if err != nil { + t.Fatal(err) + } + + if err = rss[0].Do(false, func(rec []interface{}) (bool, error) { + t.Logf("%v", rec) + m++ + return true, nil + }); err != nil { + t.Fatal(err) + } + + if g, e := n, m; g != e { + t.Fatal(g, e) + } +} + +var benchmarkCrossJoinOnce = map[string]sync.Once{} + +func benchmarkCrossJoin(b *testing.B, db *DB, create, sel List, size1, size2 int, index bool, teardown func()) { + if testing.Verbose() { + benchProlog(b) + id := fmt.Sprintf("%t|%d|%d|%t", db.isMem, size1, size2, index) + once := benchmarkCrossJoinOnce[id] + once.Do(func() { + s := "INDEXED " + if !index { + s = "NON " + s + } + b.Logf(`Fill two %stables with %d and %d records of random numbers [0, 1). Measure the performance of +%s +`, s, size1, size2, sel) + }) + benchmarkCrossJoinOnce[id] = once + } + + if teardown != nil { + defer teardown() + } + + ctx := NewRWCtx() + if _, _, err := db.Execute(ctx, create); err != nil { + b.Fatal(err) + } + + if index { + if _, _, err := db.Execute(ctx, xjoinX); err != nil { + b.Fatal(err) + } + } + + rng := rand.New(rand.NewSource(42)) + for i := 0; i < size1; i++ { + if _, _, err := db.Execute(ctx, xjoinT, rng.Float64()); err != nil { + b.Fatal(err) + } + } + for i := 0; i < size2; i++ { + if _, _, err := db.Execute(ctx, xjoinU, rng.Float64()); err != nil { + b.Fatal(err) + } + } + + if _, _, err := db.Run(ctx, "COMMIT"); err != nil { + b.Fatal(err) + } + + rs, _, err := db.Execute(nil, sel) + if err != nil { + b.Fatal(err) + } + + var n int + debug.FreeOSMemory() + b.ResetTimer() + for i := 0; i < b.N; i++ { + n = 0 + if err := rs[0].Do(false, func(rec []interface{}) (bool, error) { + n++ + return true, nil + }); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.SetBytes(int64(n) * benchScale) +} + +var ( + xjoinCreate = MustCompile(`BEGIN TRANSACTION; + CREATE TABLE t (f float); + CREATE TABLE u (f float);`) + xjoinSel = MustCompile(`SELECT * FROM (SELECT f FROM t WHERE f < 0.1), (SELECT f FROM u where f < 0.1);`) + xjoinT = MustCompile("INSERT INTO t VALUES($1);") + xjoinU = MustCompile("INSERT INTO u VALUES($1);") + xjoinX = MustCompile(`CREATE INDEX x ON t (f); CREATE INDEX y ON u (f);`) +) + +func benchmarkCrossJoinMem(b *testing.B, size1, size2 int, index bool) { + db, err := OpenMem() + if err != nil { + b.Fatal(err) + } + + benchmarkCrossJoin(b, db, xjoinCreate, xjoinSel, size1, size2, index, nil) +} + +func benchmarkCrossJoinFile(b *testing.B, size1, size2 int, index bool) { + dir, err := ioutil.TempDir("", "ql-bench-") + if err != nil { + b.Fatal(err) + } + + n := runtime.GOMAXPROCS(0) + db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true}) + if err != nil { + b.Fatal(err) + } + + benchmarkCrossJoin(b, db, xjoinCreate, xjoinSel, size1, size2, index, func() { + runtime.GOMAXPROCS(n) + db.Close() + os.RemoveAll(dir) + }) +} + +func BenchmarkCrossJoinMem1e1NoX1e2(b *testing.B) { + benchmarkCrossJoinMem(b, 1e1, 1e2, false) +} + +func BenchmarkCrossJoinMem1e1X1e2(b *testing.B) { + benchmarkCrossJoinMem(b, 1e1, 1e2, true) +} + +func BenchmarkCrossJoinMem1e2NoX1e3(b *testing.B) { + benchmarkCrossJoinMem(b, 1e2, 1e3, false) +} + +func BenchmarkCrossJoinMem1e2X1e3(b *testing.B) { + benchmarkCrossJoinMem(b, 1e2, 1e3, true) +} + +func BenchmarkCrossJoinMem1e3NoX1e4(b *testing.B) { + benchmarkCrossJoinMem(b, 1e3, 1e4, false) +} + +func BenchmarkCrossJoinMem1e3X1e4(b *testing.B) { + benchmarkCrossJoinMem(b, 1e3, 1e4, true) +} + +func BenchmarkCrossJoinMem1e2NoX1e1(b *testing.B) { + benchmarkCrossJoinMem(b, 1e2, 1e1, false) +} + +func BenchmarkCrossJoinMem1e2X1e1(b *testing.B) { + benchmarkCrossJoinMem(b, 1e2, 1e1, true) +} + +func BenchmarkCrossJoinMem1e3NoX1e2(b *testing.B) { + benchmarkCrossJoinMem(b, 1e3, 1e2, false) +} + +func BenchmarkCrossJoinMem1e3X1e2(b *testing.B) { + benchmarkCrossJoinMem(b, 1e3, 1e2, true) +} + +func BenchmarkCrossJoinMem1e4NoX1e3(b *testing.B) { + benchmarkCrossJoinMem(b, 1e4, 1e3, false) +} + +func BenchmarkCrossJoinMem1e4X1e3(b *testing.B) { + benchmarkCrossJoinMem(b, 1e4, 1e3, true) +} + +// ---- + +func BenchmarkCrossJoinFile1e1NoX1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e1, 1e2, false) +} + +func BenchmarkCrossJoinFile1e1X1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e1, 1e2, true) +} + +func BenchmarkCrossJoinFile1e2NoX1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e3, false) +} + +func BenchmarkCrossJoinFile1e2X1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e3, true) +} + +func BenchmarkCrossJoinFile1e3NoX1e4(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e4, false) +} + +func BenchmarkCrossJoinFile1e3X1e4(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e4, true) +} + +func BenchmarkCrossJoinFile1e2NoX1e1(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e1, false) +} + +func BenchmarkCrossJoinFile1e2X1e1(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e1, true) +} + +func BenchmarkCrossJoinFile1e3NoX1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e2, false) +} + +func BenchmarkCrossJoinFile1e3X1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e2, true) +} + +func BenchmarkCrossJoinFile1e4NoX1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e4, 1e3, false) +} + +func BenchmarkCrossJoinFile1e4X1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e4, 1e3, true) +} + +func TestIssue35(t *testing.T) { + var bigInt big.Int + var bigRat big.Rat + bigInt.SetInt64(42) + bigRat.SetInt64(24) + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + ctx := NewRWCtx() + _, _, err = db.Run(ctx, ` + BEGIN TRANSACTION; + CREATE TABLE t (i bigint, r bigrat); + INSERT INTO t VALUES ($1, $2); + COMMIT; + `, bigInt, bigRat) + if err != nil { + t.Fatal(err) + } + + bigInt.SetInt64(420) + bigRat.SetInt64(240) + + rs, _, err := db.Run(nil, "SELECT * FROM t;") + if err != nil { + t.Fatal(err) + } + + n := 0 + if err := rs[0].Do(false, func(rec []interface{}) (bool, error) { + switch n { + case 0: + n++ + if g, e := fmt.Sprint(rec), "[42 24/1]"; g != e { + t.Fatal(g, e) + } + + return true, nil + default: + t.Fatal(n) + panic("unreachable") + } + }); err != nil { + t.Fatal(err) + } +} + +func TestIssue28(t *testing.T) { + RegisterDriver() + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + pth := filepath.Join(dir, "ql.db") + sdb, err := sql.Open("ql", "file://"+pth) + if err != nil { + t.Fatal(err) + } + + defer sdb.Close() + tx, err := sdb.Begin() + if err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec("CREATE TABLE t (i int);"); err != nil { + t.Fatal(err) + } + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + + if _, err = os.Stat(pth); err != nil { + t.Fatal(err) + } + + pth = filepath.Join(dir, "mem.db") + mdb, err := sql.Open("ql", "memory://"+pth) + if err != nil { + t.Fatal(err) + } + + defer mdb.Close() + if tx, err = mdb.Begin(); err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec("CREATE TABLE t (i int);"); err != nil { + t.Fatal(err) + } + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + + if _, err = os.Stat(pth); err == nil { + t.Fatal(err) + } +} + +func TestIsPossiblyRewriteableCrossJoinWhereExpression(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + table := []struct { + q string + e bool + slist string + }{ + // 0 + {"SELECT * FROM t WHERE !c", false, ""}, + {"SELECT * FROM t WHERE !t.c && 4 < !u.c", true, "!c|4 3", false, ""}, + // 10 + {"SELECT * FROM t WHERE c", false, ""}, + {"SELECT * FROM t WHERE false == !t.c", true, "false==!c"}, //TODO(indices) support !c relOp fixedValue (rewrite false==!c -> !c, true==c -> c, true != c -> !c, etc.) + {"SELECT * FROM t WHERE false == ^t.c", false, ""}, + {"SELECT * FROM t WHERE false == t.c", true, "false==c"}, + {"SELECT * FROM t WHERE t.c && 4 < u.c", true, "c|4 0", true, "c|c|c>0"}, + {"SELECT * FROM t WHERE t.c && u.c && v.c", true, "c|c|c"}, + {"SELECT * FROM t WHERE t.c && u.c > 0 && v.c > 0", true, "c|c>0|c>0"}, + {"SELECT * FROM t WHERE t.c && u.c", true, "c|c"}, + // 20 + {"SELECT * FROM t WHERE t.c < 3 && u.c > 2 && v.c != 42", true, "c<3|c>2|c!=42"}, + {"SELECT * FROM t WHERE t.c > 0 && u.c && v.c", true, "c>0|c|c"}, + {"SELECT * FROM t WHERE t.c > 0 && u.c > 0 && v.c > 0", true, "c>0|c>0|c>0"}, + {"SELECT * FROM t WHERE t.c > 3 && 4 < u.c", true, "c>3|4 3 && u.c", true, "c>3|c"}, + // 25 + {"SELECT * FROM t WHERE t.c > 3", true, "c>3"}, + {"SELECT * FROM t WHERE t.c", true, "c"}, + {"SELECT * FROM t WHERE u.c == !t.c", false, ""}, + {"SELECT * FROM t WHERE u.c == 42", true, "c==42"}, + {"SELECT * FROM t WHERE u.c == ^t.c", false, ""}, + } + + for i, test := range table { + q, e, list := test.q, test.e, strings.Split(test.slist, "|") + sort.Strings(list) + l, err := Compile(q) + if err != nil { + t.Fatalf("%s\n%v", q, err) + } + + rs, _, err := db.Execute(nil, l) + if err != nil { + t.Fatalf("%s\n%v", q, err) + } + + r := rs[0].(recordset) + sel := r.rset.(*selectRset) + where := sel.src.(*whereRset) + g, glist := isPossiblyRewriteableCrossJoinWhereExpression(where.expr) + if g != e { + t.Fatalf("%d: %sg: %v e: %v", i, l, g, e) + } + + if !g { + continue + } + + a := []string{} + for _, v := range glist { + a = append(a, v.expr.String()) + } + sort.Strings(a) + if g, e := len(glist), len(list); g != e { + t.Fatalf("%d: g: %v, e: %v", i, glist, list) + } + + for j, g := range a { + if e := list[j]; g != e { + t.Fatalf("%d[%d]: g: %v e: %v", i, j, g, e) + } + } + } +} + +func dumpFields(f []*fld) string { + a := []string{} + for _, v := range f { + a = append(a, fmt.Sprintf("%p: %q", v, v.name)) + } + return strings.Join(a, ", ") +} + +func rndBytes(n int, seed int64) []byte { + rng := rand.New(rand.NewSource(seed)) + b := make([]byte, n) + for i := range b { + b[i] = byte(rng.Int()) + } + return b +} + +func TestIssue50(t *testing.T) { // https://github.com/cznic/ql/issues/50 + const dbFileName = "scans.qldb" + + type Scan struct { + ID int + Jobname string + Timestamp time.Time + Data []byte + + X, Y, Z float64 + Alpha, Beta, Gamma float64 + } + + // querys + const dbCreateTables = ` +CREATE TABLE IF NOT EXISTS Scans ( + X float, + Y float, + Z float, + Alpha float, + Beta float, + Gamma float, + Timestamp time, + Jobname string, + Data blob +); +CREATE INDEX IF NOT EXISTS ScansId on Scans (id()); +` + + const dbInsertScan = ` +INSERT INTO Scans (Timestamp,Jobname,X,Y,Z,Alpha,Beta,Gamma,Data) VALUES( +$1, +$2, +$3,$4,$5, +$6,$7,$8, +$9 +); +` + + const dbSelectOverview = `SELECT id() as ID, Jobname, Timestamp, Data, Y,Z, Gamma From Scans;` + + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create db + t.Log("Opening db.") + RegisterDriver() + db, err := sql.Open("ql", filepath.Join(dir, dbFileName)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + tx, err := db.Begin() + if err != nil { + return + } + _, err = tx.Exec(dbCreateTables) + if err != nil { + t.Fatal("could not create Table.", err) + } + + err = tx.Commit() + if err != nil { + t.Fatal("could not commit transaction.", err) + } + + // insert some data + tx, err = db.Begin() + if err != nil { + t.Fatalf("db.Begin() Error - %v", err) + } + + stmt, err := tx.Prepare(dbInsertScan) + if err != nil { + t.Fatalf("tx.Prepare(dbInsertScan) Error - %v", err) + } + defer stmt.Close() + + scanFnames := []string{"1.xyz", "2.xyz", "3.xyz"} + for _, fname := range scanFnames { + scanData, err := ioutil.ReadFile(filepath.Join("_testdata", fname)) + if err != nil { + t.Fatalf("ioutil.ReadFile(%s) Error - %v", fname, err) + } + + // hash before insert + h := md5.New() + h.Write(scanData) + + t.Logf("md5 of %s: %x", fname, h.Sum(nil)) + + _, err = stmt.Exec(time.Now(), "Job-0815", 1.0, 2.0, 3.0, 0.1, 0.2, 0.3, scanData) + if err != nil { + t.Fatalf("stmt.Exec() Error - %v", err) + return + } + } + + err = tx.Commit() + if err != nil { + t.Fatalf("tx.Commit() Error - %v", err) + } + + // select the inserted data + rows, err := db.Query(dbSelectOverview) + if err != nil { + t.Fatalf("db.Query(dbSelectOverview) Error - %v", err) + } + defer rows.Close() + + var scans []Scan + for rows.Next() { + var s Scan + var data []byte + + err = rows.Scan(&s.ID, &s.Jobname, &s.Timestamp, &data, &s.Y, &s.Z, &s.Gamma) + if err != nil { + t.Fatalf("rows.Scan(&s..) Error - %v", err) + } + scans = append(scans, s) + + // hash after select + h := md5.New() + h.Write(data) + + t.Logf("md5 of %d: %x", s.ID, h.Sum(nil)) + } + + err = rows.Err() + if err != nil { + t.Fatalf("rows.Err() Error - %v", err) + } + + t.Log("Done:", scans) +} + +func TestIssue56(t *testing.T) { + var schema = ` +CREATE TABLE IF NOT EXISTS Test ( + A string, + B string, + Suppressed bool, +); +CREATE INDEX IF NOT EXISTS aIdx ON Test (A); +CREATE INDEX IF NOT EXISTS bIdx ON Test (B); +` + + RegisterDriver() + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + pth := filepath.Join(dir, "test.db") + db, err := sql.Open("ql", "file://"+pth) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + + _, err = tx.Exec(schema) + if err != nil { + t.Fatal(err) + } + + err = tx.Commit() + if err != nil { + t.Fatal(err) + } + + // Open a new transaction and do a SELECT + + tx, err = db.Begin() + if err != nil { + t.Fatal(err) + } + + var id int64 + row := tx.QueryRow("SELECT * FROM Test") + err = row.Scan(&id) // <-- Blocks here + + switch err { + case sql.ErrNoRows: + break + case nil: + break + default: + t.Fatal(err) + } + + tx.Rollback() + return +} + +func TestRecordSetRows(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + rss, _, err := db.Run(NewRWCtx(), ` + BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (1), (2), (3), (4), (5); + COMMIT; + SELECT * FROM t ORDER BY i; + `) + if err != nil { + t.Fatal(err) + } + + tab := []struct { + limit, offset int + result []int64 + }{ + // 0 + {-1, 0, []int64{1, 2, 3, 4, 5}}, + {0, 0, nil}, + {1, 0, []int64{1}}, + {2, 0, []int64{1, 2}}, + {3, 0, []int64{1, 2, 3}}, + // 5 + {4, 0, []int64{1, 2, 3, 4}}, + {5, 0, []int64{1, 2, 3, 4, 5}}, + {6, 0, []int64{1, 2, 3, 4, 5}}, + {-1, 0, []int64{1, 2, 3, 4, 5}}, + {-1, 1, []int64{2, 3, 4, 5}}, + // 10 + {-1, 2, []int64{3, 4, 5}}, + {-1, 3, []int64{4, 5}}, + {-1, 4, []int64{5}}, + {-1, 5, nil}, + {3, 0, []int64{1, 2, 3}}, + // 15 + {3, 1, []int64{2, 3, 4}}, + {3, 2, []int64{3, 4, 5}}, + {3, 3, []int64{4, 5}}, + {3, 4, []int64{5}}, + {3, 5, nil}, + // 20 + {-1, 2, []int64{3, 4, 5}}, + {0, 2, nil}, + {1, 2, []int64{3}}, + {2, 2, []int64{3, 4}}, + {3, 2, []int64{3, 4, 5}}, + // 25 + {4, 2, []int64{3, 4, 5}}, + } + + rs := rss[0] + for iTest, test := range tab { + t.Log(iTest) + rows, err := rs.Rows(test.limit, test.offset) + if err != nil { + t.Fatal(err) + } + + if g, e := len(rows), len(test.result); g != e { + t.Log(rows, test.result) + t.Fatal(g, e) + } + + for i, row := range rows { + if g, e := len(row), 1; g != e { + t.Fatal(i, g, i) + } + + if g, e := row[0], test.result[i]; g != e { + t.Fatal(i, g, e) + } + } + } +} + +func TestRecordFirst(t *testing.T) { + q := MustCompile("SELECT * FROM t WHERE i > $1 ORDER BY i;") + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + if _, _, err = db.Run(NewRWCtx(), ` + BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (1), (2), (3), (4), (5); + COMMIT; + `); err != nil { + t.Fatal(err) + } + + tab := []struct { + par int64 + result int64 + }{ + {-1, 1}, + {0, 1}, + {1, 2}, + {2, 3}, + {3, 4}, + {4, 5}, + {5, -1}, + } + + for iTest, test := range tab { + t.Log(iTest) + rss, _, err := db.Execute(nil, q, test.par) + if err != nil { + t.Fatal(err) + } + + row, err := rss[0].FirstRow() + if err != nil { + t.Fatal(err) + } + + switch { + case test.result < 0: + if row != nil { + t.Fatal(row) + } + default: + if row == nil { + t.Fatal(row) + } + + if g, e := len(row), 1; g != e { + t.Fatal(g, e) + } + + if g, e := row[0], test.result; g != e { + t.Fatal(g, e) + } + } + } +} + +var issue63 = MustCompile(` +BEGIN TRANSACTION; + CREATE TABLE Forecast (WeatherProvider string, Timestamp time, MinTemp int32, MaxTemp int32); + INSERT INTO Forecast VALUES ("dwd.de", now(), 20, 22); +COMMIT; +SELECT * FROM Forecast WHERE Timestamp > 0;`) + +func TestIssue63(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + rs, _, err := db.Execute(NewRWCtx(), issue63) + if err != nil { + t.Fatal(err) + } + + if _, err = rs[0].Rows(-1, 0); err == nil { + t.Fatal(err) + } + + t.Log(err) + if g, e := strings.Contains(err.Error(), "invalid operation"), true; g != e { + t.Fatal(g, e) + } + + if g, e := strings.Contains(err.Error(), "mismatched types time.Time and int64"), true; g != e { + t.Fatal(g, e) + } +} + +const issue66Src = ` +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE UNIQUE INDEX x ON t (i); + INSERT INTO t VALUES (1), (1); +COMMIT;` + +var issue66 = MustCompile(issue66Src) + +func TestIssue66Mem(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + + _, _, err = db.Execute(NewRWCtx(), issue66) + if err == nil { + t.Fatal(err) + } + + t.Log(err) +} + +func TestIssue66File(t *testing.T) { + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + + db, err := OpenFile(filepath.Join(dir, "test.db"), &Options{CanCreate: true}) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + _, _, err = db.Execute(NewRWCtx(), issue66) + if err == nil { + t.Fatal(err) + } + + t.Log(err) +} + +func TestIssue66MemDriver(t *testing.T) { + RegisterMemDriver() + db, err := sql.Open("ql-mem", "TestIssue66MemDriver-"+fmt.Sprintf("%d", time.Now().UnixNano())) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec(issue66Src); err == nil { + t.Fatal(err) + } + + t.Log(err) +} + +func TestIssue66FileDriver(t *testing.T) { + RegisterDriver() + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + + db, err := sql.Open("ql", filepath.Join(dir, "TestIssue66MemDriver")) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec(issue66Src); err == nil { + t.Fatal(err) + } + + t.Log(err) +} + +func Example_lIKE() { + db, err := OpenMem() + if err != nil { + panic(err) + } + + rss, _, err := db.Run(NewRWCtx(), ` + BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES + (1, "seafood"), + (2, "A fool on the hill"), + (3, NULL), + (4, "barbaz"), + (5, "foobar"), + ; + COMMIT; + + SELECT * FROM t WHERE s LIKE "foo" ORDER BY i; + SELECT * FROM t WHERE s LIKE "^bar" ORDER BY i; + SELECT * FROM t WHERE s LIKE "bar$" ORDER BY i; + SELECT * FROM t WHERE !(s LIKE "foo") ORDER BY i;`, + ) + if err != nil { + panic(err) + } + + for _, rs := range rss { + if err := rs.Do(false, func(data []interface{}) (bool, error) { + fmt.Println(data) + return true, nil + }); err != nil { + panic(err) + } + fmt.Println("----") + } + // Output: + // [1 seafood] + // [2 A fool on the hill] + // [5 foobar] + // ---- + // [4 barbaz] + // ---- + // [5 foobar] + // ---- + // [4 barbaz] + // ---- +} + +func TestIssue73(t *testing.T) { + RegisterDriver() + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + pth := filepath.Join(dir, "test.db") + + for i := 0; i < 10; i++ { + var db *sql.DB + var tx *sql.Tx + var err error + var row *sql.Row + var name string + + if db, err = sql.Open("ql", pth); err != nil { + t.Fatal("sql.Open: ", err) + } + + t.Log("Call to db.Begin()") + if tx, err = db.Begin(); err != nil { + t.Fatal("db.Begin: ", err) + } + + t.Log("Call to tx.QueryRow()") + row = tx.QueryRow(`SELECT Name FROM __Table`) + t.Log("Call to tx.Commit()") + if err = tx.Commit(); err != nil { + t.Fatal("tx.Commit: ", err) + } + + row.Scan(&name) + t.Log("name: ", name) + } +} + +func Example_id() { + db, err := OpenMem() + if err != nil { + panic(err) + } + + rss, _, err := db.Run(NewRWCtx(), ` + BEGIN TRANSACTION; + CREATE TABLE foo (i int); + INSERT INTO foo VALUES (10), (20); + CREATE TABLE bar (fooID int, s string); + INSERT INTO bar SELECT id(), "ten" FROM foo WHERE i == 10; + INSERT INTO bar SELECT id(), "twenty" FROM foo WHERE i == 20; + COMMIT; + SELECT * + FROM foo, bar + WHERE bar.fooID == id(foo) + ORDER BY id(foo);`, + ) + if err != nil { + panic(err) + } + + for _, rs := range rss { + if err := rs.Do(false, func(data []interface{}) (bool, error) { + fmt.Println(data) + return true, nil + }); err != nil { + panic(err) + } + fmt.Println("----") + } + // Output: + // [10 1 ten] + // [20 2 twenty] + // ---- +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/benchcmp b/Godeps/_workspace/src/github.com/cznic/ql/benchcmp new file mode 100644 index 00000000000..a0f7a50c08d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/benchcmp @@ -0,0 +1 @@ +go test -test.run NONE -test.bench . -test.benchmem -timeout 1h | tee $1 diff --git a/Godeps/_workspace/src/github.com/cznic/ql/blob.go b/Godeps/_workspace/src/github.com/cznic/ql/blob.go new file mode 100644 index 00000000000..0aec6a959f5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/blob.go @@ -0,0 +1,140 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "encoding/gob" + "log" + "math/big" + "sync" + "time" +) + +const shortBlob = 256 // bytes + +var ( + gobInitDuration = time.Duration(278) + gobInitInt = big.NewInt(42) + gobInitRat = big.NewRat(355, 113) + gobInitTime time.Time +) + +func init() { + var err error + if gobInitTime, err = time.ParseInLocation( + "Jan 2, 2006 at 3:04pm (MST)", + "Jul 9, 2012 at 5:02am (CEST)", + time.FixedZone("XYZ", 1234), + ); err != nil { + log.Panic(err) + } + newGobCoder() +} + +type gobCoder struct { + buf bytes.Buffer + dec *gob.Decoder + enc *gob.Encoder + mu sync.Mutex +} + +func newGobCoder() (g *gobCoder) { + g = &gobCoder{} + g.enc = gob.NewEncoder(&g.buf) + if err := g.enc.Encode(gobInitInt); err != nil { + log.Panic(err) + } + + if err := g.enc.Encode(gobInitRat); err != nil { + log.Panic(err) + } + + if err := g.enc.Encode(gobInitTime); err != nil { + log.Panic(err) + } + + if err := g.enc.Encode(gobInitDuration); err != nil { + log.Panic(err) + } + + g.dec = gob.NewDecoder(&g.buf) + i := big.NewInt(0) + if err := g.dec.Decode(i); err != nil { + log.Panic(err) + } + + r := big.NewRat(3, 5) + if err := g.dec.Decode(r); err != nil { + log.Panic(err) + } + + t := time.Now() + if err := g.dec.Decode(&t); err != nil { + log.Panic(err) + } + + var d time.Duration + if err := g.dec.Decode(&d); err != nil { + log.Panic(err) + } + + return +} + +func (g *gobCoder) encode(v interface{}) (b []byte, err error) { + g.mu.Lock() + defer g.mu.Unlock() + + g.buf.Reset() + switch x := v.(type) { + case []byte: + return x, nil + case *big.Int: + err = g.enc.Encode(x) + case *big.Rat: + err = g.enc.Encode(x) + case time.Time: + err = g.enc.Encode(x) + case time.Duration: + err = g.enc.Encode(int64(x)) + default: + //dbg("%T(%v)", v, v) + log.Panic("internal error 002") + } + b = g.buf.Bytes() + return +} + +func (g *gobCoder) decode(b []byte, typ int) (v interface{}, err error) { + g.mu.Lock() + defer g.mu.Unlock() + + g.buf.Reset() + g.buf.Write(b) + switch typ { + case qBlob: + return b, nil + case qBigInt: + x := big.NewInt(0) + err = g.dec.Decode(&x) + v = x + case qBigRat: + x := big.NewRat(1, 1) + err = g.dec.Decode(&x) + v = x + case qTime: + var x time.Time + err = g.dec.Decode(&x) + v = x + case qDuration: + var x int64 + err = g.dec.Decode(&x) + v = time.Duration(x) + default: + log.Panic("internal error 003") + } + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/btree.go b/Godeps/_workspace/src/github.com/cznic/ql/btree.go new file mode 100644 index 00000000000..4f6423902b7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/btree.go @@ -0,0 +1,725 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "io" +) + +const ( + kx = 128 //DONE benchmark tune this number if using custom key/value type(s). + kd = 64 //DONE benchmark tune this number if using custom key/value type(s). +) + +type ( + // cmp compares a and b. Return value is: + // + // < 0 if a < b + // 0 if a == b + // > 0 if a > b + // + cmp func(a, b []interface{}) int + + d struct { // data page + c int + d [2*kd + 1]de + n *d + p *d + } + + de struct { // d element + k []interface{} + v []interface{} + } + + enumerator struct { + err error + hit bool + i int + k []interface{} + q *d + t *tree + ver int64 + } + + // tree is a B+tree. + tree struct { + c int + cmp cmp + first *d + last *d + r interface{} + ver int64 + } + + xe struct { // x element + ch interface{} + sep *d + } + + x struct { // index page + c int + x [2*kx + 2]xe + } +) + +var ( // R/O zero values + zd d + zde de + zx x + zxe xe +) + +func clr(q interface{}) { + switch x := q.(type) { + case *x: + for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn + clr(x.x[i].ch) + } + *x = zx // GC + case *d: + *x = zd // GC + } +} + +// -------------------------------------------------------------------------- x + +func newX(ch0 interface{}) *x { + r := &x{} + r.x[0].ch = ch0 + return r +} + +func (q *x) extract(i int) { + q.c-- + if i < q.c { + copy(q.x[i:], q.x[i+1:q.c+1]) + q.x[q.c].ch = q.x[q.c+1].ch + q.x[q.c].sep = nil // GC + q.x[q.c+1] = zxe // GC + } +} + +func (q *x) insert(i int, d *d, ch interface{}) *x { + c := q.c + if i < c { + q.x[c+1].ch = q.x[c].ch + copy(q.x[i+2:], q.x[i+1:c]) + q.x[i+1].sep = q.x[i].sep + } + c++ + q.c = c + q.x[i].sep = d + q.x[i+1].ch = ch + return q +} + +func (q *x) siblings(i int) (l, r *d) { + if i >= 0 { + if i > 0 { + l = q.x[i-1].ch.(*d) + } + if i < q.c { + r = q.x[i+1].ch.(*d) + } + } + return +} + +// -------------------------------------------------------------------------- d + +func (l *d) mvL(r *d, c int) { + copy(l.d[l.c:], r.d[:c]) + copy(r.d[:], r.d[c:r.c]) + l.c += c + r.c -= c +} + +func (l *d) mvR(r *d, c int) { + copy(r.d[c:], r.d[:r.c]) + copy(r.d[:c], l.d[l.c-c:]) + r.c += c + l.c -= c +} + +// ----------------------------------------------------------------------- tree + +// treeNew returns a newly created, empty tree. The compare function is used +// for key collation. +func treeNew(cmp cmp) *tree { + return &tree{cmp: cmp} +} + +// Clear removes all K/V pairs from the tree. +func (t *tree) Clear() { + if t.r == nil { + return + } + + clr(t.r) + t.c, t.first, t.last, t.r = 0, nil, nil, nil + t.ver++ +} + +func (t *tree) cat(p *x, q, r *d, pi int) { + t.ver++ + q.mvL(r, r.c) + if r.n != nil { + r.n.p = q + } else { + t.last = q + } + q.n = r.n + if p.c > 1 { + p.extract(pi) + p.x[pi].ch = q + } else { + t.r = q + } +} + +func (t *tree) catX(p, q, r *x, pi int) { + t.ver++ + q.x[q.c].sep = p.x[pi].sep + copy(q.x[q.c+1:], r.x[:r.c]) + q.c += r.c + 1 + q.x[q.c].ch = r.x[r.c].ch + if p.c > 1 { + p.c-- + pc := p.c + if pi < pc { + p.x[pi].sep = p.x[pi+1].sep + copy(p.x[pi+1:], p.x[pi+2:pc+1]) + p.x[pc].ch = p.x[pc+1].ch + p.x[pc].sep = nil // GC + p.x[pc+1].ch = nil // GC + } + return + } + + t.r = q +} + +//Delete removes the k's KV pair, if it exists, in which case Delete returns +//true. +func (t *tree) Delete(k []interface{}) (ok bool) { + pi := -1 + var p *x + q := t.r + if q == nil { + return + } + + for { + var i int + i, ok = t.find(q, k) + if ok { + switch x := q.(type) { + case *x: + dp := x.x[i].sep + switch { + case dp.c > kd: + t.extract(dp, 0) + default: + if x.c < kx && q != t.r { + t.underflowX(p, &x, pi, &i) + } + pi = i + 1 + p = x + q = x.x[pi].ch + ok = false + continue + } + case *d: + t.extract(x, i) + if x.c >= kd { + return + } + + if q != t.r { + t.underflow(p, x, pi) + } else if t.c == 0 { + t.Clear() + } + } + return + } + + switch x := q.(type) { + case *x: + if x.c < kx && q != t.r { + t.underflowX(p, &x, pi, &i) + } + pi = i + p = x + q = x.x[i].ch + case *d: + return + } + } +} + +func (t *tree) extract(q *d, i int) { // (r []interface{}) { + t.ver++ + //r = q.d[i].v // prepared for Extract + q.c-- + if i < q.c { + copy(q.d[i:], q.d[i+1:q.c+1]) + } + q.d[q.c] = zde // GC + t.c-- + return +} + +func (t *tree) find(q interface{}, k []interface{}) (i int, ok bool) { + var mk []interface{} + l := 0 + switch x := q.(type) { + case *x: + h := x.c - 1 + for l <= h { + m := (l + h) >> 1 + mk = x.x[m].sep.d[0].k + switch cmp := t.cmp(k, mk); { + case cmp > 0: + l = m + 1 + case cmp == 0: + return m, true + default: + h = m - 1 + } + } + case *d: + h := x.c - 1 + for l <= h { + m := (l + h) >> 1 + mk = x.d[m].k + switch cmp := t.cmp(k, mk); { + case cmp > 0: + l = m + 1 + case cmp == 0: + return m, true + default: + h = m - 1 + } + } + } + return l, false +} + +// First returns the first item of the tree in the key collating order, or +// (nil, nil) if the tree is empty. +func (t *tree) First() (k []interface{}, v []interface{}) { + if q := t.first; q != nil { + q := &q.d[0] + k, v = q.k, q.v + } + return +} + +// Get returns the value associated with k and true if it exists. Otherwise Get +// returns (nil, false). +func (t *tree) Get(k []interface{}) (v []interface{}, ok bool) { + q := t.r + if q == nil { + return + } + + for { + var i int + if i, ok = t.find(q, k); ok { + switch x := q.(type) { + case *x: + return x.x[i].sep.d[0].v, true + case *d: + return x.d[i].v, true + } + } + switch x := q.(type) { + case *x: + q = x.x[i].ch + default: + return + } + } +} + +func (t *tree) insert(q *d, i int, k []interface{}, v []interface{}) *d { + t.ver++ + c := q.c + if i < c { + copy(q.d[i+1:], q.d[i:c]) + } + c++ + q.c = c + q.d[i].k, q.d[i].v = k, v + t.c++ + return q +} + +// Last returns the last item of the tree in the key collating order, or (nil, +// nil) if the tree is empty. +func (t *tree) Last() (k []interface{}, v []interface{}) { + if q := t.last; q != nil { + q := &q.d[q.c-1] + k, v = q.k, q.v + } + return +} + +// Len returns the number of items in the tree. +func (t *tree) Len() int { + return t.c +} + +func (t *tree) overflow(p *x, q *d, pi, i int, k []interface{}, v []interface{}) { + t.ver++ + l, r := p.siblings(pi) + + if l != nil && l.c < 2*kd { + l.mvL(q, 1) + t.insert(q, i-1, k, v) + return + } + + if r != nil && r.c < 2*kd { + if i < 2*kd { + q.mvR(r, 1) + t.insert(q, i, k, v) + } else { + t.insert(r, 0, k, v) + } + return + } + + t.split(p, q, pi, i, k, v) +} + +// Seek returns an enumerator positioned on a an item such that k >= item's +// key. ok reports if k == item.key The enumerator's position is possibly +// after the last item in the tree. +func (t *tree) Seek(k []interface{}) (e *enumerator, ok bool) { + q := t.r + if q == nil { + e = &enumerator{nil, false, 0, k, nil, t, t.ver} + return + } + + for { + var i int + if i, ok = t.find(q, k); ok { + switch x := q.(type) { + case *x: + e = &enumerator{nil, ok, 0, k, x.x[i].sep, t, t.ver} + return + case *d: + e = &enumerator{nil, ok, i, k, x, t, t.ver} + return + } + } + switch x := q.(type) { + case *x: + q = x.x[i].ch + case *d: + e = &enumerator{nil, ok, i, k, x, t, t.ver} + return + } + } +} + +// SeekFirst returns an enumerator positioned on the first KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returned and e will be nil. +func (t *tree) SeekFirst() (e *enumerator, err error) { + q := t.first + if q == nil { + return nil, io.EOF + } + + return &enumerator{nil, true, 0, q.d[0].k, q, t, t.ver}, nil +} + +// SeekLast returns an enumerator positioned on the last KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returned and e will be nil. +func (t *tree) SeekLast() (e *enumerator, err error) { + q := t.last + if q == nil { + return nil, io.EOF + } + + return &enumerator{nil, true, q.c - 1, q.d[q.c-1].k, q, t, t.ver}, nil +} + +// Set sets the value associated with k. +func (t *tree) Set(k []interface{}, v []interface{}) { + pi := -1 + var p *x + q := t.r + if q != nil { + for { + i, ok := t.find(q, k) + if ok { + switch x := q.(type) { + case *x: + x.x[i].sep.d[0].v = v + case *d: + x.d[i].v = v + } + return + } + + switch x := q.(type) { + case *x: + if x.c > 2*kx { + t.splitX(p, &x, pi, &i) + } + pi = i + p = x + q = x.x[i].ch + case *d: + switch { + case x.c < 2*kd: + t.insert(x, i, k, v) + default: + t.overflow(p, x, pi, i, k, v) + } + return + } + } + } + + z := t.insert(&d{}, 0, k, v) + t.r, t.first, t.last = z, z, z + return +} + +func (t *tree) split(p *x, q *d, pi, i int, k []interface{}, v []interface{}) { + t.ver++ + r := &d{} + if q.n != nil { + r.n = q.n + r.n.p = r + } else { + t.last = r + } + q.n = r + r.p = q + + copy(r.d[:], q.d[kd:2*kd]) + for i := range q.d[kd:] { + q.d[kd+i] = zde + } + q.c = kd + r.c = kd + if pi >= 0 { + p.insert(pi, r, r) + } else { + t.r = newX(q).insert(0, r, r) + } + if i > kd { + t.insert(r, i-kd, k, v) + return + } + + t.insert(q, i, k, v) +} + +func (t *tree) splitX(p *x, pp **x, pi int, i *int) { + t.ver++ + q := *pp + r := &x{} + copy(r.x[:], q.x[kx+1:]) + q.c = kx + r.c = kx + if pi >= 0 { + p.insert(pi, q.x[kx].sep, r) + } else { + t.r = newX(q).insert(0, q.x[kx].sep, r) + } + q.x[kx].sep = nil + for i := range q.x[kx+1:] { + q.x[kx+i+1] = zxe + } + if *i > kx { + *pp = r + *i -= kx + 1 + } +} + +func (t *tree) underflow(p *x, q *d, pi int) { + t.ver++ + l, r := p.siblings(pi) + + if l != nil && l.c+q.c >= 2*kd { + l.mvR(q, 1) + } else if r != nil && q.c+r.c >= 2*kd { + q.mvL(r, 1) + r.d[r.c] = zde // GC + } else if l != nil { + t.cat(p, l, q, pi-1) + } else { + t.cat(p, q, r, pi) + } +} + +func (t *tree) underflowX(p *x, pp **x, pi int, i *int) { + t.ver++ + var l, r *x + q := *pp + + if pi >= 0 { + if pi > 0 { + l = p.x[pi-1].ch.(*x) + } + if pi < p.c { + r = p.x[pi+1].ch.(*x) + } + } + + if l != nil && l.c > kx { + q.x[q.c+1].ch = q.x[q.c].ch + copy(q.x[1:], q.x[:q.c]) + q.x[0].ch = l.x[l.c].ch + q.x[0].sep = p.x[pi-1].sep + q.c++ + *i++ + l.c-- + p.x[pi-1].sep = l.x[l.c].sep + return + } + + if r != nil && r.c > kx { + q.x[q.c].sep = p.x[pi].sep + q.c++ + q.x[q.c].ch = r.x[0].ch + p.x[pi].sep = r.x[0].sep + copy(r.x[:], r.x[1:r.c]) + r.c-- + rc := r.c + r.x[rc].ch = r.x[rc+1].ch + r.x[rc].sep = nil + r.x[rc+1].ch = nil + return + } + + if l != nil { + *i += l.c + 1 + t.catX(p, l, q, pi-1) + *pp = l + return + } + + t.catX(p, q, r, pi) +} + +// ----------------------------------------------------------------- enumerator + +// Next returns the currently enumerated item, if it exists and moves to the +// next item in the key collation order. If there is no item to return, err == +// io.EOF is returned. +func (e *enumerator) Next() (k []interface{}, v []interface{}, err error) { + if err = e.err; err != nil { + return + } + + if e.ver != e.t.ver { + f, hit := e.t.Seek(e.k) + if !e.hit && hit { + if err = f.next(); err != nil { + return + } + } + + *e = *f + } + if e.q == nil { + e.err, err = io.EOF, io.EOF + return + } + + if e.i >= e.q.c { + if err = e.next(); err != nil { + return + } + } + + i := e.q.d[e.i] + k, v = i.k, i.v + e.k, e.hit = k, false + e.next() + return +} + +func (e *enumerator) next() error { + if e.q == nil { + e.err = io.EOF + return io.EOF + } + + switch { + case e.i < e.q.c-1: + e.i++ + default: + if e.q, e.i = e.q.n, 0; e.q == nil { + e.err = io.EOF + } + } + return e.err +} + +// Prev returns the currently enumerated item, if it exists and moves to the +// previous item in the key collation order. If there is no item to return, err +// == io.EOF is returned. +func (e *enumerator) Prev() (k []interface{}, v []interface{}, err error) { + if err = e.err; err != nil { + return + } + + if e.ver != e.t.ver { + f, hit := e.t.Seek(e.k) + if !e.hit && hit { + if err = f.prev(); err != nil { + return + } + } + + *e = *f + } + if e.q == nil { + e.err, err = io.EOF, io.EOF + return + } + + if e.i >= e.q.c { + if err = e.next(); err != nil { + return + } + } + + i := e.q.d[e.i] + k, v = i.k, i.v + e.k, e.hit = k, false + e.prev() + return +} + +func (e *enumerator) prev() error { + if e.q == nil { + e.err = io.EOF + return io.EOF + } + + switch { + case e.i > 0: + e.i-- + default: + if e.q = e.q.p; e.q == nil { + e.err = io.EOF + break + } + + e.i = e.q.c - 1 + } + return e.err +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/builtin.go b/Godeps/_workspace/src/github.com/cznic/ql/builtin.go new file mode 100644 index 00000000000..45081934b6b --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/builtin.go @@ -0,0 +1,878 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "fmt" + "log" + "math/rand" + "reflect" + "strings" + "time" +) + +//TODO agg bigint, bigrat, time, duration + +var builtin = map[string]struct { + f func([]interface{}, map[interface{}]interface{}) (interface{}, error) + minArgs int + maxArgs int + isStatic bool + isAggregate bool +}{ + "__testBlob": {builtinTestBlob, 1, 1, true, false}, + "__testString": {builtinTestString, 1, 1, true, false}, + "avg": {builtinAvg, 1, 1, false, true}, + "complex": {builtinComplex, 2, 2, true, false}, + "contains": {builtinContains, 2, 2, true, false}, + "count": {builtinCount, 0, 1, false, true}, + "date": {builtinDate, 8, 8, true, false}, + "day": {builtinDay, 1, 1, true, false}, + "formatTime": {builtinFormatTime, 2, 2, true, false}, + "hasPrefix": {builtinHasPrefix, 2, 2, true, false}, + "hasSuffix": {builtinHasSuffix, 2, 2, true, false}, + "hour": {builtinHour, 1, 1, true, false}, + "hours": {builtinHours, 1, 1, true, false}, + "id": {builtinID, 0, 1, false, false}, + "imag": {builtinImag, 1, 1, true, false}, + "len": {builtinLen, 1, 1, true, false}, + "max": {builtinMax, 1, 1, false, true}, + "min": {builtinMin, 1, 1, false, true}, + "minute": {builtinMinute, 1, 1, true, false}, + "minutes": {builtinMinutes, 1, 1, true, false}, + "month": {builtinMonth, 1, 1, true, false}, + "nanosecond": {builtinNanosecond, 1, 1, true, false}, + "nanoseconds": {builtinNanoseconds, 1, 1, true, false}, + "now": {builtinNow, 0, 0, false, false}, + "parseTime": {builtinParseTime, 2, 2, true, false}, + "real": {builtinReal, 1, 1, true, false}, + "second": {builtinSecond, 1, 1, true, false}, + "seconds": {builtinSeconds, 1, 1, true, false}, + "since": {builtinSince, 1, 1, false, false}, + "sum": {builtinSum, 1, 1, false, true}, + "timeIn": {builtinTimeIn, 2, 2, true, false}, + "weekday": {builtinWeekday, 1, 1, true, false}, + "year": {builtinYear, 1, 1, true, false}, + "yearDay": {builtinYearday, 1, 1, true, false}, +} + +func badNArgs(min int, s string, arg []interface{}) error { + a := []string{} + for _, v := range arg { + a = append(a, fmt.Sprintf("%v", v)) + } + switch len(arg) < min { + case true: + return fmt.Errorf("missing argument to %s(%s)", s, strings.Join(a, ", ")) + default: //case false: + return fmt.Errorf("too many arguments to %s(%s)", s, strings.Join(a, ", ")) + } +} + +func invArg(arg interface{}, s string) error { + return fmt.Errorf("invalid argument %v (type %T) for %s", arg, arg, s) +} + +func builtinTestBlob(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + n, err := intExpr(arg[0]) + if err != nil { + return nil, err + } + + rng := rand.New(rand.NewSource(n)) + b := make([]byte, n) + for i := range b { + b[i] = byte(rng.Int()) + } + return b, nil +} + +func builtinTestString(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + n, err := intExpr(arg[0]) + if err != nil { + return nil, err + } + + rng := rand.New(rand.NewSource(n)) + b := make([]byte, n) + for i := range b { + b[i] = byte(rng.Int()) + } + return string(b), nil +} + +func builtinAvg(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + type avg struct { + sum interface{} + n uint64 + } + + if _, ok := ctx["$agg0"]; ok { + return + } + + fn := ctx["$fn"] + if _, ok := ctx["$agg"]; ok { + data, ok := ctx[fn].(avg) + if !ok { + return + } + + switch x := data.sum.(type) { + case complex64: + return complex64(complex128(x) / complex(float64(data.n), 0)), nil + case complex128: + return complex64(complex128(x) / complex(float64(data.n), 0)), nil + case float32: + return float32(float64(x) / float64(data.n)), nil + case float64: + return float64(x) / float64(data.n), nil + case int8: + return int8(int64(x) / int64(data.n)), nil + case int16: + return int16(int64(x) / int64(data.n)), nil + case int32: + return int32(int64(x) / int64(data.n)), nil + case int64: + return int64(int64(x) / int64(data.n)), nil + case uint8: + return uint8(uint64(x) / data.n), nil + case uint16: + return uint16(uint64(x) / data.n), nil + case uint32: + return uint32(uint64(x) / data.n), nil + case uint64: + return uint64(uint64(x) / data.n), nil + } + + } + + data, _ := ctx[fn].(avg) + y := arg[0] + if y == nil { + return + } + + switch x := data.sum.(type) { + case nil: + switch y := y.(type) { + case float32, float64, int8, int16, int32, int64, uint8, uint16, uint32, uint64: + data = avg{y, 0} + default: + return nil, fmt.Errorf("avg: cannot accept %v (value if type %T)", y, y) + } + case complex64: + data.sum = x + y.(complex64) + case complex128: + data.sum = x + y.(complex128) + case float32: + data.sum = x + y.(float32) + case float64: + data.sum = x + y.(float64) + case int8: + data.sum = x + y.(int8) + case int16: + data.sum = x + y.(int16) + case int32: + data.sum = x + y.(int32) + case int64: + data.sum = x + y.(int64) + case uint8: + data.sum = x + y.(uint8) + case uint16: + data.sum = x + y.(uint16) + case uint32: + data.sum = x + y.(uint32) + case uint64: + data.sum = x + y.(uint64) + } + data.n++ + ctx[fn] = data + return +} + +func builtinComplex(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + re, im := arg[0], arg[1] + if re == nil || im == nil { + return nil, nil + } + + re, im = coerce(re, im) + if reflect.TypeOf(re) != reflect.TypeOf(im) { + return nil, fmt.Errorf("complex(%T(%#v), %T(%#v)): invalid types", re, re, im, im) + } + + switch re := re.(type) { + case idealFloat: + return idealComplex(complex(float64(re), float64(im.(idealFloat)))), nil + case idealInt: + return idealComplex(complex(float64(re), float64(im.(idealInt)))), nil + case idealRune: + return idealComplex(complex(float64(re), float64(im.(idealRune)))), nil + case idealUint: + return idealComplex(complex(float64(re), float64(im.(idealUint)))), nil + case float32: + return complex(float32(re), im.(float32)), nil + case float64: + return complex(float64(re), im.(float64)), nil + case int8: + return complex(float64(re), float64(im.(int8))), nil + case int16: + return complex(float64(re), float64(im.(int16))), nil + case int32: + return complex(float64(re), float64(im.(int32))), nil + case int64: + return complex(float64(re), float64(im.(int64))), nil + case uint8: + return complex(float64(re), float64(im.(uint8))), nil + case uint16: + return complex(float64(re), float64(im.(uint16))), nil + case uint32: + return complex(float64(re), float64(im.(uint32))), nil + case uint64: + return complex(float64(re), float64(im.(uint64))), nil + default: + return nil, invArg(re, "complex") + } +} + +func builtinContains(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + switch s := arg[0].(type) { + case nil: + return nil, nil + case string: + switch chars := arg[1].(type) { + case nil: + return nil, nil + case string: + return strings.Contains(s, chars), nil + default: + return nil, invArg(chars, "string") + } + default: + return nil, invArg(s, "string") + } +} + +func builtinCount(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + if _, ok := ctx["$agg0"]; ok { + return int64(0), nil + } + + fn := ctx["$fn"] + if _, ok := ctx["$agg"]; ok { + return ctx[fn].(int64), nil + } + + n, _ := ctx[fn].(int64) + switch len(arg) { + case 0: + n++ + case 1: + if arg[0] != nil { + n++ + } + default: + log.Panic("internal error 067") + } + ctx[fn] = n + return +} + +func builtinDate(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + for i, v := range arg { + switch i { + case 7: + switch x := v.(type) { + case string: + default: + return nil, invArg(x, "date") + } + default: + switch x := v.(type) { + case int64: + case idealInt: + arg[i] = int64(x) + default: + return nil, invArg(x, "date") + } + } + } + + sloc := arg[7].(string) + loc := time.Local + switch sloc { + case "local": + default: + loc, err = time.LoadLocation(sloc) + if err != nil { + return + } + } + + return time.Date( + int(arg[0].(int64)), + time.Month(arg[1].(int64)), + int(arg[2].(int64)), + int(arg[3].(int64)), + int(arg[4].(int64)), + int(arg[5].(int64)), + int(arg[6].(int64)), + loc, + ), nil +} + +func builtinLen(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case string: + return int64(len(x)), nil + default: + return nil, invArg(x, "len") + } +} + +func builtinDay(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Day()), nil + default: + return nil, invArg(x, "day") + } +} + +func builtinFormatTime(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + switch y := arg[1].(type) { + case nil: + return nil, nil + case string: + return x.Format(y), nil + default: + return nil, invArg(y, "formatTime") + } + default: + return nil, invArg(x, "formatTime") + } +} + +func builtinHasPrefix(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + switch s := arg[0].(type) { + case nil: + return nil, nil + case string: + switch prefix := arg[1].(type) { + case nil: + return nil, nil + case string: + return strings.HasPrefix(s, prefix), nil + default: + return nil, invArg(prefix, "string") + } + default: + return nil, invArg(s, "string") + } +} + +func builtinHasSuffix(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + switch s := arg[0].(type) { + case nil: + return nil, nil + case string: + switch suffix := arg[1].(type) { + case nil: + return nil, nil + case string: + return strings.HasSuffix(s, suffix), nil + default: + return nil, invArg(suffix, "string") + } + default: + return nil, invArg(s, "string") + } +} + +func builtinHour(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Hour()), nil + default: + return nil, invArg(x, "hour") + } +} + +func builtinHours(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Duration: + return x.Hours(), nil + default: + return nil, invArg(x, "hours") + } +} + +func builtinID(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := ctx["$id"].(type) { + case map[string]interface{}: + if len(arg) == 0 { + return nil, nil + } + + tab := arg[0].(*ident) + id, ok := x[tab.s] + if !ok { + return nil, fmt.Errorf("value not available: id(%s)", tab) + } + + if _, ok := id.(int64); ok { + return id, nil + } + + return nil, fmt.Errorf("value not available: id(%s)", tab) + case int64: + return x, nil + default: + panic("internal error 072") + } +} + +func builtinImag(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case idealComplex: + return imag(x), nil + case complex64: + return imag(x), nil + case complex128: + return imag(x), nil + default: + return nil, invArg(x, "imag") + } +} + +func builtinMax(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + if _, ok := ctx["$agg0"]; ok { + return + } + + fn := ctx["$fn"] + if _, ok := ctx["$agg"]; ok { + if v, ok = ctx[fn]; ok { + return + } + + return nil, nil + } + + max := ctx[fn] + y := arg[0] + if y == nil { + return + } + switch x := max.(type) { + case nil: + switch y := y.(type) { + case float32, float64, string, int8, int16, int32, int64, uint8, uint16, uint32, uint64, time.Time: + max = y + default: + return nil, fmt.Errorf("max: cannot accept %v (value if type %T)", y, y) + } + case float32: + if y := y.(float32); y > x { + max = y + } + case float64: + if y := y.(float64); y > x { + max = y + } + case string: + if y := y.(string); y > x { + max = y + } + case int8: + if y := y.(int8); y > x { + max = y + } + case int16: + if y := y.(int16); y > x { + max = y + } + case int32: + if y := y.(int32); y > x { + max = y + } + case int64: + if y := y.(int64); y > x { + max = y + } + case uint8: + if y := y.(uint8); y > x { + max = y + } + case uint16: + if y := y.(uint16); y > x { + max = y + } + case uint32: + if y := y.(uint32); y > x { + max = y + } + case uint64: + if y := y.(uint64); y > x { + max = y + } + case time.Time: + if y := y.(time.Time); y.After(x) { + max = y + } + } + ctx[fn] = max + return +} + +func builtinMin(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + if _, ok := ctx["$agg0"]; ok { + return + } + + fn := ctx["$fn"] + if _, ok := ctx["$agg"]; ok { + if v, ok = ctx[fn]; ok { + return + } + + return nil, nil + } + + min := ctx[fn] + y := arg[0] + if y == nil { + return + } + switch x := min.(type) { + case nil: + switch y := y.(type) { + case float32, float64, string, int8, int16, int32, int64, uint8, uint16, uint32, uint64, time.Time: + min = y + default: + return nil, fmt.Errorf("min: cannot accept %v (value if type %T)", y, y) + } + case float32: + if y := y.(float32); y < x { + min = y + } + case float64: + if y := y.(float64); y < x { + min = y + } + case string: + if y := y.(string); y < x { + min = y + } + case int8: + if y := y.(int8); y < x { + min = y + } + case int16: + if y := y.(int16); y < x { + min = y + } + case int32: + if y := y.(int32); y < x { + min = y + } + case int64: + if y := y.(int64); y < x { + min = y + } + case uint8: + if y := y.(uint8); y < x { + min = y + } + case uint16: + if y := y.(uint16); y < x { + min = y + } + case uint32: + if y := y.(uint32); y < x { + min = y + } + case uint64: + if y := y.(uint64); y < x { + min = y + } + case time.Time: + if y := y.(time.Time); y.Before(x) { + min = y + } + } + ctx[fn] = min + return +} + +func builtinMinute(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Minute()), nil + default: + return nil, invArg(x, "minute") + } +} + +func builtinMinutes(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Duration: + return x.Minutes(), nil + default: + return nil, invArg(x, "minutes") + } +} + +func builtinMonth(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Month()), nil + default: + return nil, invArg(x, "month") + } +} + +func builtinNanosecond(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Nanosecond()), nil + default: + return nil, invArg(x, "nanosecond") + } +} + +func builtinNanoseconds(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Duration: + return x.Nanoseconds(), nil + default: + return nil, invArg(x, "nanoseconds") + } +} + +func builtinNow(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + return time.Now(), nil +} + +func builtinParseTime(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + var a [2]string + for i, v := range arg { + switch x := v.(type) { + case nil: + return nil, nil + case string: + a[i] = x + default: + return nil, invArg(x, "parseTime") + } + } + + t, err := time.Parse(a[0], a[1]) + if err != nil { + return nil, err + } + + ls := t.Location().String() + if ls == "UTC" { + return t, nil + } + + l, err := time.LoadLocation(ls) + if err != nil { + return t, nil + } + + return time.ParseInLocation(a[0], a[1], l) +} + +func builtinReal(arg []interface{}, _ map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case idealComplex: + return real(x), nil + case complex64: + return real(x), nil + case complex128: + return real(x), nil + default: + return nil, invArg(x, "real") + } +} + +func builtinSecond(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Second()), nil + default: + return nil, invArg(x, "second") + } +} + +func builtinSeconds(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Duration: + return x.Seconds(), nil + default: + return nil, invArg(x, "seconds") + } +} + +func builtinSince(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return time.Since(x), nil + default: + return nil, invArg(x, "since") + } +} + +func builtinSum(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + if _, ok := ctx["$agg0"]; ok { + return + } + + fn := ctx["$fn"] + if _, ok := ctx["$agg"]; ok { + if v, ok = ctx[fn]; ok { + return + } + + return nil, nil + } + + sum := ctx[fn] + y := arg[0] + if y == nil { + return + } + switch x := sum.(type) { + case nil: + switch y := y.(type) { + case complex64, complex128, float32, float64, int8, int16, int32, int64, uint8, uint16, uint32, uint64: + sum = y + default: + return nil, fmt.Errorf("sum: cannot accept %v (value if type %T)", y, y) + } + case complex64: + sum = x + y.(complex64) + case complex128: + sum = x + y.(complex128) + case float32: + sum = x + y.(float32) + case float64: + sum = x + y.(float64) + case int8: + sum = x + y.(int8) + case int16: + sum = x + y.(int16) + case int32: + sum = x + y.(int32) + case int64: + sum = x + y.(int64) + case uint8: + sum = x + y.(uint8) + case uint16: + sum = x + y.(uint16) + case uint32: + sum = x + y.(uint32) + case uint64: + sum = x + y.(uint64) + } + ctx[fn] = sum + return +} + +func builtinTimeIn(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + switch y := arg[1].(type) { + case nil: + return nil, nil + case string: + loc := time.Local + switch y { + case "local": + default: + loc, err = time.LoadLocation(y) + if err != nil { + return + } + } + + return x.In(loc), nil + default: + return nil, invArg(x, "timeIn") + } + default: + return nil, invArg(x, "timeIn") + } +} + +func builtinWeekday(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Weekday()), nil + default: + return nil, invArg(x, "weekday") + } +} + +func builtinYear(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.Year()), nil + default: + return nil, invArg(x, "year") + } +} + +func builtinYearday(arg []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + switch x := arg[0].(type) { + case nil: + return nil, nil + case time.Time: + return int64(x.YearDay()), nil + default: + return nil, invArg(x, "yearDay") + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/coerce.go b/Godeps/_workspace/src/github.com/cznic/ql/coerce.go new file mode 100644 index 00000000000..368d5b38980 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/coerce.go @@ -0,0 +1,290 @@ +// Copyright 2013 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: This file was generated automatically by +// +// $ go run helper -o coerce.go +// +// DO NOT EDIT! + +package ql + +import ( + "math" + "math/big" + "reflect" + "time" +) + +func coerce(a, b interface{}) (x, y interface{}) { + if reflect.TypeOf(a) == reflect.TypeOf(b) { + return a, b + } + + switch a.(type) { + case idealComplex, idealFloat, idealInt, idealRune, idealUint: + switch b.(type) { + case idealComplex, idealFloat, idealInt, idealRune, idealUint: + x, y = coerce1(a, b), b + if reflect.TypeOf(x) == reflect.TypeOf(y) { + return + } + + return a, coerce1(b, a) + default: + return coerce1(a, b), b + } + default: + switch b.(type) { + case idealComplex, idealFloat, idealInt, idealRune, idealUint: + return a, coerce1(b, a) + default: + return a, b + } + } +} + +func coerce1(inVal, otherVal interface{}) (coercedInVal interface{}) { + coercedInVal = inVal + if otherVal == nil { + return + } + + switch x := inVal.(type) { + case nil: + return + case idealComplex: + switch otherVal.(type) { + //case idealComplex: + //case idealFloat: + //case idealInt: + //case idealRune: + //case idealUint: + //case bool: + case complex64: + return complex64(x) + case complex128: + return complex128(x) + //case float32: + //case float64: + //case int8: + //case int16: + //case int32: + //case int64: + //case string: + //case uint8: + //case uint16: + //case uint32: + //case uint64: + //case *big.Int: + //case *big.Rat: + //case time.Time: + //case time.Duration: + } + case idealFloat: + switch otherVal.(type) { + case idealComplex: + return idealComplex(complex(float64(x), 0)) + case idealFloat: + return idealFloat(float64(x)) + //case idealInt: + //case idealRune: + //case idealUint: + //case bool: + case complex64: + return complex64(complex(float32(x), 0)) + case complex128: + return complex128(complex(float64(x), 0)) + case float32: + return float32(float64(x)) + case float64: + return float64(float64(x)) + //case int8: + //case int16: + //case int32: + //case int64: + //case string: + //case uint8: + //case uint16: + //case uint32: + //case uint64: + //case *big.Int: + case *big.Rat: + return big.NewRat(1, 1).SetFloat64(float64(x)) + //case time.Time: + //case time.Duration: + } + case idealInt: + switch otherVal.(type) { + case idealComplex: + return idealComplex(complex(float64(x), 0)) + case idealFloat: + return idealFloat(int64(x)) + case idealInt: + return idealInt(int64(x)) + //case idealRune: + case idealUint: + if x >= 0 { + return idealUint(int64(x)) + } + //case bool: + case complex64: + return complex64(complex(float32(x), 0)) + case complex128: + return complex128(complex(float64(x), 0)) + case float32: + return float32(int64(x)) + case float64: + return float64(int64(x)) + case int8: + if x >= math.MinInt8 && x <= math.MaxInt8 { + return int8(int64(x)) + } + case int16: + if x >= math.MinInt16 && x <= math.MaxInt16 { + return int16(int64(x)) + } + case int32: + if x >= math.MinInt32 && x <= math.MaxInt32 { + return int32(int64(x)) + } + case int64: + return int64(int64(x)) + //case string: + case uint8: + if x >= 0 && x <= math.MaxUint8 { + return uint8(int64(x)) + } + case uint16: + if x >= 0 && x <= math.MaxUint16 { + return uint16(int64(x)) + } + case uint32: + if x >= 0 && x <= math.MaxUint32 { + return uint32(int64(x)) + } + case uint64: + if x >= 0 { + return uint64(int64(x)) + } + case *big.Int: + return big.NewInt(int64(x)) + case *big.Rat: + return big.NewRat(1, 1).SetInt64(int64(x)) + //case time.Time: + case time.Duration: + return time.Duration(int64(x)) + } + case idealRune: + switch otherVal.(type) { + case idealComplex: + return idealComplex(complex(float64(x), 0)) + case idealFloat: + return idealFloat(int64(x)) + case idealInt: + return idealInt(int64(x)) + case idealRune: + return idealRune(int64(x)) + case idealUint: + return idealUint(int64(x)) + //case bool: + case complex64: + return complex64(complex(float32(x), 0)) + case complex128: + return complex128(complex(float64(x), 0)) + case float32: + return float32(int64(x)) + case float64: + return float64(int64(x)) + case int8: + return int8(int64(x)) + case int16: + return int16(int64(x)) + case int32: + return int32(int64(x)) + case int64: + return int64(int64(x)) + //case string: + case uint8: + return uint8(int64(x)) + case uint16: + return uint16(int64(x)) + case uint32: + return uint32(int64(x)) + case uint64: + return uint64(int64(x)) + case *big.Int: + return big.NewInt(int64(x)) + case *big.Rat: + return big.NewRat(1, 1).SetInt64(int64(x)) + //case time.Time: + case time.Duration: + return time.Duration(int64(x)) + } + case idealUint: + switch otherVal.(type) { + case idealComplex: + return idealComplex(complex(float64(x), 0)) + case idealFloat: + return idealFloat(uint64(x)) + case idealInt: + if x <= math.MaxInt64 { + return idealInt(int64(x)) + } + //case idealRune: + case idealUint: + return idealUint(uint64(x)) + //case bool: + case complex64: + return complex64(complex(float32(x), 0)) + case complex128: + return complex128(complex(float64(x), 0)) + case float32: + return float32(uint64(x)) + case float64: + return float64(uint64(x)) + case int8: + if x <= math.MaxInt8 { + return int8(int64(x)) + } + case int16: + if x <= math.MaxInt16 { + return int16(int64(x)) + } + case int32: + if x <= math.MaxInt32 { + return int32(int64(x)) + } + case int64: + if x <= math.MaxInt64 { + return int64(int64(x)) + } + //case string: + case uint8: + if x >= 0 && x <= math.MaxUint8 { + return uint8(int64(x)) + } + case uint16: + if x >= 0 && x <= math.MaxUint16 { + return uint16(int64(x)) + } + case uint32: + if x >= 0 && x <= math.MaxUint32 { + return uint32(int64(x)) + } + case uint64: + return uint64(uint64(x)) + case *big.Int: + return big.NewInt(0).SetUint64(uint64(x)) + case *big.Rat: + return big.NewRat(1, 1).SetInt(big.NewInt(0).SetUint64(uint64(x))) + //case time.Time: + case time.Duration: + if x <= math.MaxInt64 { + return time.Duration(int64(x)) + } + } + } + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/design/doc.go b/Godeps/_workspace/src/github.com/cznic/ql/design/doc.go new file mode 100644 index 00000000000..6e94c6c8a5e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/design/doc.go @@ -0,0 +1,298 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Package design describes some of the data structures used in QL. + +Handles + +A handle is a 7 byte "pointer" to a block in the DB[0]. + +Scalar encoding + +Encoding of so called "scalars" provided by [1]. Unless specified otherwise, +all values discussed below are scalars, encoded scalars or encoding of scalar +arrays. + +Database root + +DB root is a 1-scalar found at a fixed handle (#1). + + +---+------+--------+-----------------------+ + | # | Name | Type | Description | + +---+------+--------+-----------------------+ + | 0 | head | handle | First table meta data | + +---+------+--------+-----------------------+ + +Head is the head of a single linked list of table of meta data. It's zero if +there are no tables in the DB. + +Table meta data + +Table meta data are a 6-scalar. + + +---+---------+--------+--------------------------+ + | # | Name | Type | Description | + +---+---------+--------+--------------------------+ + | 0 | next | handle | Next table meta data. | + | 1 | scols | string | Column defintitions | + | 2 | hhead | handle | -> head -> first record | + | 3 | name | string | Table name | + | 4 | indices | string | Index definitions | + | 5 | hxroots | handle | Index B+Trees roots list | + +---+---------+--------+--------------------------+ + +Fields #4 and #5 are optional for backward compatibility with existing +databases. OTOH, forward compatibility will not work. Once any indices are +created using a newer QL version the older versions of QL, expecting only 4 +fields of meta data will not be able to use the DB. That's the intended +behavior because the older versions of QL cannot update the indexes, which can +break queries runned by the newer QL version which expect indices to be always +actualized on any table-with-indices mutation. + +The handle of the next table meta data is in the field #0 (next). If there is +no next table meta data, the field is zero. Names and types of table columns +are stored in field #1 (scols). A single field is described by concatenating a +type tag and the column name. The type tags are + + bool 'b' + complex64 'c' + complex128 'd' + float32 'f' + float64 'g', alias float + int8 'i' + int16 'j' + int32 'k' + int64 'l', alias int + string 's' + uint8 'u', alias byte + uint16 'v' + uint32 'w' + uint64 'x', alias uint + bigInt 'I' + bigRat 'R' + blob 'B' + duration 'D' + time 'T' + +The scols value is the above described encoded fields joined using "|". For +example + + CREATE TABLE t (Foo bool, Bar string, Baz float); + +This statement adds a table meta data with scols + + "bFool|sBar|gBaz" + +Columns can be dropped from a table + + ALTER TABLE t DROP COLUMN Bar; + +This "erases" the field info in scols, so the value becomes + + "bFool||gBaz" + +Colums can be added to a table + + ALTER TABLE t ADD Count uint; + +New fields are always added to the end of scols + + "bFool||gBaz|xCount" + +Index of a field in strings.Split(scols, "|") is the index of the field in a +table record. The above discussed rules for column dropping and column adding +allow for schema evolution without a need to reshape any existing table data. +Dropped columns are left where they are and new records insert nil in their +place. The encoded nil is one byte. Added columns, when not present in +preexisting records are returned as nil values. If the overhead of dropped +columns becomes an issue and there's time/space and memory enough to move the +records of a table around: + + BEGIN TRANSACTION; + CREATE TABLE new (column definitions); + INSERT INTO new SELECT * FROM old; + DROP TABLE old; + CREATE TABLE old (column definitions); + INSERT INTO old SELECT * FROM new; + DROP TABLE new; + END TRANSACTION; + +This is not very time/space effective and for Big Data it can cause an OOM +because transactions are limited by memory resources available to the process. +Perhaps a method and/or QL statement to do this in-place should be added +(MAYBE consider adopting MySQL's OPTIMIZE TABLE syntax). + +Field #2 (hhead) is a handle to a head of table records, i.e. not a handle to +the first record in the table. It is thus always non zero even for a table +having no records. The reason for this "double pointer" schema is to enable +adding (linking) a new record by updating a single value of the (hhead pointing +to) head. + + tableMeta.hhead -> head -> firstTableRecord + +The table name is stored in field #3 (name). + +Indices + +Consider an index named N, indexing column named C. The encoding of this +particular index is a string "N". is a string "n" for non unique +indices and "u" for unique indices. There is this index information for the +index possibly indexing the record id() and for all other columns of scols. +Where the column is not indexed, the index info is an empty string. Infos for +all indexes are joined with "|". For example + + BEGIN TRANSACTION; + CREATE TABLE t (Foo int, Bar bool, Baz string); + CREATE INDEX X ON t (Baz); + CREATE UNIQUE INDEX Y ON t (Foo); + COMMIT; + +The values of fields #1 and #4 for the above are + + scols: "lFoo|bBar|sBaz" + indices: "|uY||nX" + +Aligning properly the "|" split parts + + id col #0 col#1 col#2 + +----------+----+--------+--------+--------+ + | scols: | | "lFoo" | "bBar" | "sBaz" | + +----------+----+--------+--------+--------+ + | indices: | "" | "uY" | "" | "nX" | + +----------+----+--------+--------+--------+ + +shows that the record id() is not indexed for this table while the columns Foo +and Baz are. + +Note that there cannot be two differently named indexes for the same column and +it's intended. The indices are B+Trees[2]. The list of handles to their roots +is pointed to by hxroots with zeros for non indexed columns. For the previous +example + + tableMeta.hxroots -> {0, y, 0, x} + +where x is the root of the B+Tree for the X index and y is the root of the +B+Tree for the Y index. If there would be an index for id(), its B+Tree root +will be present where the first zero is. Similarly to hhead, hxroots is never +zero, even when there are no indices for a table. + +Table record + +A table record is an N-scalar. + + +-----+------------+--------+-------------------------------+ + | # | Name | Type | Description | + +-----+------------+--------+-------------------------------+ + | 0 | next | handle | Next record or zero. | + | 1 | id | int64 | Automatically assigned unique | + | | | | value obtainable by id(). | + | 2 | field #0 | scalar | First field of the record. | + | 3 | field #1 | scalar | Second field of the record. | + ... + | N-1 | field #N-2 | scalar | Last field of the record. | + +-----+------------+--------+-------------------------------+ + +The linked "ordering" of table records has no semantics and it doesn't have to +correlate to the order of how the records were added to the table. In fact, an +efficient way of the linking leads to "ordering" which is actually reversed wrt +the insertion order. + +Non unique index + +The composite key of the B+Tree is {indexed value, record handle}. The B+Tree +value is not used. + + B+Tree key B+Tree value + +---------------+---------------+ +--------------+ + | Indexed Value | Record Handle | -> | not used | + +---------------+---------------+ +--------------+ + +Unique index + +If the indexed value is NULL then the composite B+Tree key is {nil, record +handle} and the B+Tree value is not used. + + B+Tree key B+Tree value + +------+-----------------+ +--------------+ + | NULL | Record Handle | -> | not used | + +------+-----------------+ +--------------+ + +If the indexed value is not NULL then key of the B+Tree key is the indexed +value and the B+Tree value is the record handle. + + B+Tree key B+Tree value + +------------------------+ +---------------+ + | Non NULL Indexed Value | -> | Record Handle | + +------------------------+ +---------------+ + +Non scalar types + +Scalar types of [1] are bool, complex*, float*, int*, uint*, string and []byte +types. All other types are "blob-like". + + QL type Go type + ----------------------------- + blob []byte + bigint big.Int + bigrat big.Rat + time time.Time + duration time.Duration + +Memory back-end stores the Go type directly. File back-end must resort to +encode all of the above as (tagged) []byte due to the lack of more types +supported natively by lldb. NULL values of blob-like types are encoded as nil +(gbNull in lldb/gb.go), exactly the same as the already existing QL types are. + +Blob encoding + +The values of the blob-like types are first encoded into a []byte slice: + + +-----------------------+-------------------+ + | blob | raw | + | bigint, bigrat, time | gob encoded | + | duration | gob encoded int64 | + +-----------------------+-------------------+ + +The gob encoding is "differential" wrt an initial encoding of all of the +blob-like type. IOW, the initial type descriptors which gob encoding must write +out are stripped off and "resupplied" on decoding transparently. See also +blob.go. If the length of the resulting slice is <= shortBlob, the first and +only chunk is the scalar encoding of + + + []interface{}{typeTag, slice}. // initial (and last) chunk + +The length of slice can be zero (for blob("")). If the resulting slice is long +(> shortBlob), the first chunk comes from encoding + + []interface{}{typeTag, nextHandle, firstPart}. // initial, but not final chunk + +In this case len(firstPart) <= shortBlob. Second and other chunks: If the chunk +is the last one, src is + + []interface{lastPart}. // overflow chunk (last) + +In this case len(lastPart) <= 64kB. If the chunk is not the last one, src is + + []interface{}{nextHandle, part}. // overflow chunk (not last) + +In this case len(part) == 64kB. + +Links + +Referenced from above: + + [0]: http://godoc.org/github.com/cznic/exp/lldb#hdr-Block_handles + [1]: http://godoc.org/github.com/cznic/exp/lldb#EncodeScalars + [2]: http://godoc.org/github.com/cznic/exp/lldb#BTree + +Rationale + +While these notes might be useful to anyone looking at QL sources, the +specifically intended reader is my future self. + +*/ +package design diff --git a/Godeps/_workspace/src/github.com/cznic/ql/doc.go b/Godeps/_workspace/src/github.com/cznic/ql/doc.go new file mode 100644 index 00000000000..e8e8389733a --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/doc.go @@ -0,0 +1,2295 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//MAYBE set operations +//MAYBE IN ( SELECT ... ) +//MAYBE +=, -=, ... + +//TODO verify there's a graceful failure for a 2G+ blob on a 32 bit machine. + +// Package ql is a pure Go embedded (S)QL database. +// +// QL is a SQL-like language. It is less complex and less powerful than SQL +// (whichever specification SQL is considered to be). +// +// Change list +// +// 2015-02-16: IN predicate now accepts a SELECT statement. See the updated +// "Predicates" section. +// +// 2015-01-17: Logical operators || and && have now alternative spellings: OR +// and AND (case insensitive). AND was a keyword before, but OR is a new one. +// This can possibly break existing queries. For the record, it's a good idea +// to not use any name appearing in, for example, [7] in your queries as the +// list of QL's keywords may expand for gaining better compatibility with +// existing SQL "standards". +// +// 2015-01-12: ACID guarantees were tightened at the cost of performance in +// some cases. The write collecting window mechanism, a formerly used +// implementation detail, was removed. Inserting rows one by one in a +// transaction is now slow. I mean very slow. Try to avoid inserting single +// rows in a transaction. Instead, whenever possible, perform batch updates of +// tens to, say thousands of rows in a single transaction. See also: +// http://www.sqlite.org/faq.html#q19, the discussed synchronization principles +// involved are the same as for QL, modulo minor details. +// +// Note: A side effect is that closing a DB before exiting an application, both +// for the Go API and through database/sql driver, is no more required, +// strictly speaking. Beware that exiting an application while there is an open +// (uncommitted) transaction in progress means losing the transaction data. +// However, the DB will not become corrupted because of not closing it. Nor +// that was the case before, but formerly failing to close a DB could have +// resulted in losing the data of the last transaction. +// +// 2014-09-21: id() now optionally accepts a single argument - a table name. +// +// 2014-09-01: Added the DB.Flush() method and the LIKE pattern matching +// predicate. +// +// 2014-08-08: The built in functions max and min now accept also time values. +// Thanks opennota! (https://github.com/opennota) +// +// 2014-06-05: RecordSet interface extended by new methods FirstRow and Rows. +// +// 2014-06-02: Indices on id() are now used by SELECT statements. +// +// 2014-05-07: Introduction of Marshal, Schema, Unmarshal. +// +// 2014-04-15: +// +// Added optional IF NOT EXISTS clause to CREATE INDEX and optional IF EXISTS +// clause to DROP INDEX. +// +// 2014-04-12: +// +// The column Unique in the virtual table __Index was renamed to IsUnique +// because the old name is a keyword. Unfortunately, this is a breaking change, +// sorry. +// +// 2014-04-11: Introduction of LIMIT, OFFSET. +// +// 2014-04-10: Introduction of query rewriting. +// +// 2014-04-07: Introduction of indices. +// +// Notation +// +// The syntax is specified using Extended Backus-Naur Form (EBNF) +// +// Production = production_name "=" [ Expression ] "." . +// Expression = Alternative { "|" Alternative } . +// Alternative = Term { Term } . +// Term = production_name | token [ "…" token ] | Group | Option | Repetition . +// Group = "(" Expression ")" . +// Option = "[" Expression "]" . +// Repetition = "{" Expression "}" . +// Productions are expressions constructed from terms and the following operators, in increasing precedence +// +// | alternation +// () grouping +// [] option (0 or 1 times) +// {} repetition (0 to n times) +// +// Lower-case production names are used to identify lexical tokens. +// Non-terminals are in CamelCase. Lexical tokens are enclosed in double quotes +// "" or back quotes ``. +// +// The form a … b represents the set of characters from a through b as +// alternatives. The horizontal ellipsis … is also used elsewhere in the spec +// to informally denote various enumerations or code snippets that are not +// further specified. +// +// QL source code representation +// +// QL source code is Unicode text encoded in UTF-8. The text is not +// canonicalized, so a single accented code point is distinct from the same +// character constructed from combining an accent and a letter; those are +// treated as two code points. For simplicity, this document will use the +// unqualified term character to refer to a Unicode code point in the source +// text. +// +// Each code point is distinct; for instance, upper and lower case letters are +// different characters. +// +// Implementation restriction: For compatibility with other tools, the parser +// may disallow the NUL character (U+0000) in the statement. +// +// Implementation restriction: A byte order mark is disallowed anywhere in QL +// statements. +// +// Characters +// +// The following terms are used to denote specific character classes +// +// newline = . // the Unicode code point U+000A +// unicode_char = . // an arbitrary Unicode code point except newline +// ascii_letter = "a" … "z" | "A" … "Z" . +// +// Letters and digits +// +// The underscore character _ (U+005F) is considered a letter. +// +// letter = ascii_letter | "_" . +// decimal_digit = "0" … "9" . +// octal_digit = "0" … "7" . +// hex_digit = "0" … "9" | "A" … "F" | "a" … "f" . +// +// Lexical elements +// +// Lexical elements are comments, tokens, identifiers, keywords, operators and +// delimiters, integer, floating-point, imaginary, rune and string literals and +// QL parameters. +// +// Comments +// +// There are three forms of comments +// +// Line comments start with the character sequence // or -- and stop at the end +// of the line. A line comment acts like a space. +// +// General comments start with the character sequence /* and continue through +// the character sequence */. A general comment acts like a space. +// +// Comments do not nest. +// +// Tokens +// +// Tokens form the vocabulary of QL. There are four classes: identifiers, +// keywords, operators and delimiters, and literals. White space, formed from +// spaces (U+0020), horizontal tabs (U+0009), carriage returns (U+000D), and +// newlines (U+000A), is ignored except as it separates tokens that would +// otherwise combine into a single token. +// +// Semicolons +// +// The formal grammar uses semicolons ";" as separators of QL statements. A +// single QL statement or the last QL statement in a list of statements can +// have an optional semicolon terminator. (Actually a separator from the +// following empty statement.) +// +// Identifiers +// +// Identifiers name entities such as tables or record set columns. An +// identifier is a sequence of one or more letters and digits. The first +// character in an identifier must be a letter. +// +// identifier = letter { letter | decimal_digit } . +// +// For example +// +// price +// _tmp42 +// Sales +// +// No identifiers are predeclared, however note that no keyword can be used as +// an identifier. Identifiers starting with two underscores are used for meta +// data virtual tables names. For forward compatibility, users should generally +// avoid using any identifiers starting with two underscores. For example +// +// __Column +// __Index +// __Table +// +// Keywords +// +// The following keywords are reserved and may not be used as identifiers. +// +// ADD BY duration INDEX NULL TRUNCATE +// ALTER byte EXISTS INSERT OFFSET uint +// AND COLUMN false int ON uint16 +// AS complex128 float int16 ORDER uint32 +// ASC complex64 float32 int32 SELECT uint64 +// BETWEEN CREATE float64 int64 SET uint8 +// bigint DELETE FROM int8 string UNIQUE +// bigrat DESC GROUP INTO TABLE UPDATE +// blob DISTINCT IF LIMIT time VALUES +// bool DROP IN LIKE true WHERE +// NOT OR +// +// Keywords are not case sensitive. +// +// Operators and Delimiters +// +// The following character sequences represent operators, delimiters, and other +// special tokens +// +// + & && == != ( ) +// - | || < <= [ ] +// * ^ > >= , ; +// / << = . +// % >> ! +// &^ +// +// Operators consisting of more than one character are referred to by names in +// the rest of the documentation +// +// andand = "&&" . +// andnot = "&^" . +// lsh = "<<" . +// le = "<=" . +// eq = "==" . +// ge = ">=" . +// neq = "!=" . +// oror = "||" . +// rsh = ">>" . +// +// Integer literals +// +// An integer literal is a sequence of digits representing an integer constant. +// An optional prefix sets a non-decimal base: 0 for octal, 0x or 0X for +// hexadecimal. In hexadecimal literals, letters a-f and A-F represent values +// 10 through 15. +// +// int_lit = decimal_lit | octal_lit | hex_lit . +// decimal_lit = ( "1" … "9" ) { decimal_digit } . +// octal_lit = "0" { octal_digit } . +// hex_lit = "0" ( "x" | "X" ) hex_digit { hex_digit } . +// +// For example +// +// 42 +// 0600 +// 0xBadFace +// 1701411834604692 +// +// Floating-point literals +// +// A floating-point literal is a decimal representation of a floating-point +// constant. It has an integer part, a decimal point, a fractional part, and an +// exponent part. The integer and fractional part comprise decimal digits; the +// exponent part is an e or E followed by an optionally signed decimal +// exponent. One of the integer part or the fractional part may be elided; one +// of the decimal point or the exponent may be elided. +// +// float_lit = decimals "." [ decimals ] [ exponent ] | +// decimals exponent | +// "." decimals [ exponent ] . +// decimals = decimal_digit { decimal_digit } . +// exponent = ( "e" | "E" ) [ "+" | "-" ] decimals . +// +// For example +// +// 0. +// 72.40 +// 072.40 // == 72.40 +// 2.71828 +// 1.e+0 +// 6.67428e-11 +// 1E6 +// .25 +// .12345E+5 +// +// Imaginary literals +// +// An imaginary literal is a decimal representation of the imaginary part of a +// complex constant. It consists of a floating-point literal or decimal integer +// followed by the lower-case letter i. +// +// imaginary_lit = (decimals | float_lit) "i" . +// +// For example +// +// 0i +// 011i // == 11i +// 0.i +// 2.71828i +// 1.e+0i +// 6.67428e-11i +// 1E6i +// .25i +// .12345E+5i +// +// Rune literals +// +// A rune literal represents a rune constant, an integer value identifying a +// Unicode code point. A rune literal is expressed as one or more characters +// enclosed in single quotes. Within the quotes, any character may appear +// except single quote and newline. A single quoted character represents the +// Unicode value of the character itself, while multi-character sequences +// beginning with a backslash encode values in various formats. +// +// The simplest form represents the single character within the quotes; since +// QL statements are Unicode characters encoded in UTF-8, multiple +// UTF-8-encoded bytes may represent a single integer value. For instance, the +// literal 'a' holds a single byte representing a literal a, Unicode U+0061, +// value 0x61, while 'ä' holds two bytes (0xc3 0xa4) representing a literal +// a-dieresis, U+00E4, value 0xe4. +// +// Several backslash escapes allow arbitrary values to be encoded as ASCII +// text. There are four ways to represent the integer value as a numeric +// constant: \x followed by exactly two hexadecimal digits; \u followed by +// exactly four hexadecimal digits; \U followed by exactly eight hexadecimal +// digits, and a plain backslash \ followed by exactly three octal digits. In +// each case the value of the literal is the value represented by the digits in +// the corresponding base. +// +// Although these representations all result in an integer, they have different +// valid ranges. Octal escapes must represent a value between 0 and 255 +// inclusive. Hexadecimal escapes satisfy this condition by construction. The +// escapes \u and \U represent Unicode code points so within them some values +// are illegal, in particular those above 0x10FFFF and surrogate halves. +// +// After a backslash, certain single-character escapes represent special +// values +// +// \a U+0007 alert or bell +// \b U+0008 backspace +// \f U+000C form feed +// \n U+000A line feed or newline +// \r U+000D carriage return +// \t U+0009 horizontal tab +// \v U+000b vertical tab +// \\ U+005c backslash +// \' U+0027 single quote (valid escape only within rune literals) +// \" U+0022 double quote (valid escape only within string literals) +// +// All other sequences starting with a backslash are illegal inside rune +// literals. +// +// rune_lit = "'" ( unicode_value | byte_value ) "'" . +// unicode_value = unicode_char | little_u_value | big_u_value | escaped_char . +// byte_value = octal_byte_value | hex_byte_value . +// octal_byte_value = `\` octal_digit octal_digit octal_digit . +// hex_byte_value = `\` "x" hex_digit hex_digit . +// little_u_value = `\` "u" hex_digit hex_digit hex_digit hex_digit . +// big_u_value = `\` "U" hex_digit hex_digit hex_digit hex_digit +// hex_digit hex_digit hex_digit hex_digit . +// escaped_char = `\` ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | `\` | "'" | `"` ) . +// +// For example +// +// 'a' +// 'ä' +// '本' +// '\t' +// '\000' +// '\007' +// '\377' +// '\x07' +// '\xff' +// '\u12e4' +// '\U00101234' +// 'aa' // illegal: too many characters +// '\xa' // illegal: too few hexadecimal digits +// '\0' // illegal: too few octal digits +// '\uDFFF' // illegal: surrogate half +// '\U00110000' // illegal: invalid Unicode code point +// +// String literals +// +// A string literal represents a string constant obtained from concatenating a +// sequence of characters. There are two forms: raw string literals and +// interpreted string literals. +// +// Raw string literals are character sequences between back quotes ``. Within +// the quotes, any character is legal except back quote. The value of a raw +// string literal is the string composed of the uninterpreted (implicitly +// UTF-8-encoded) characters between the quotes; in particular, backslashes +// have no special meaning and the string may contain newlines. Carriage +// returns inside raw string literals are discarded from the raw string value. +// +// Interpreted string literals are character sequences between double quotes +// "". The text between the quotes, which may not contain newlines, forms the +// value of the literal, with backslash escapes interpreted as they are in rune +// literals (except that \' is illegal and \" is legal), with the same +// restrictions. The three-digit octal (\nnn) and two-digit hexadecimal (\xnn) +// escapes represent individual bytes of the resulting string; all other +// escapes represent the (possibly multi-byte) UTF-8 encoding of individual +// characters. Thus inside a string literal \377 and \xFF represent a single +// byte of value 0xFF=255, while ÿ, \u00FF, \U000000FF and \xc3\xbf represent +// the two bytes 0xc3 0xbf of the UTF-8 encoding of character U+00FF. +// +// string_lit = raw_string_lit | interpreted_string_lit . +// raw_string_lit = "`" { unicode_char | newline } "`" . +// interpreted_string_lit = `"` { unicode_value | byte_value } `"` . +// +// For example +// +// `abc` // same as "abc" +// `\n +// \n` // same as "\\n\n\\n" +// "\n" +// "" +// "Hello, world!\n" +// "日本語" +// "\u65e5本\U00008a9e" +// "\xff\u00FF" +// "\uD800" // illegal: surrogate half +// "\U00110000" // illegal: invalid Unicode code point +// +// These examples all represent the same string +// +// "日本語" // UTF-8 input text +// `日本語` // UTF-8 input text as a raw literal +// "\u65e5\u672c\u8a9e" // the explicit Unicode code points +// "\U000065e5\U0000672c\U00008a9e" // the explicit Unicode code points +// "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" // the explicit UTF-8 bytes +// +// If the statement source represents a character as two code points, such as a +// combining form involving an accent and a letter, the result will be an error +// if placed in a rune literal (it is not a single code point), and will appear +// as two code points if placed in a string literal. +// +// QL parameters +// +// Literals are assigned their values from the respective text representation +// at "compile" (parse) time. QL parameters provide the same functionality as +// literals, but their value is assigned at execution time from an expression +// list passed to DB.Run or DB.Execute. Using '?' or '$' is completely +// equivalent. +// +// ql_parameter = ( "?" | "$" ) "1" … "9" { "0" … "9" } . +// +// For example +// +// SELECT DepartmentID +// FROM department +// WHERE DepartmentID == ?1 +// ORDER BY DepartmentName; +// +// SELECT employee.LastName +// FROM department, employee +// WHERE department.DepartmentID == $1 && employee.LastName > $2 +// ORDER BY DepartmentID; +// +// Constants +// +// Keywords 'false' and 'true' (not case sensitive) represent the two possible +// constant values of type bool (also not case sensitive). +// +// Keyword 'NULL' (not case sensitive) represents an untyped constant which is +// assignable to any type. NULL is distinct from any other value of any type. +// +// Types +// +// A type determines the set of values and operations specific to values of +// that type. A type is specified by a type name. +// +// Type = "bigint" // http://golang.org/pkg/math/big/#Int +// | "bigrat" // http://golang.org/pkg/math/big/#Rat +// | "blob" // []byte +// | "bool" +// | "byte" // alias for uint8 +// | "complex128" +// | "complex64" +// | "duration" // http://golang.org/pkg/time/#Duration +// | "float" // alias for float64 +// | "float32" +// | "float64" +// | "int" // alias for int64 +// | "int16" +// | "int32" +// | "int64" +// | "int8" +// | "rune" // alias for int32 +// | "string" +// | "time" // http://golang.org/pkg/time/#Time +// | "uint" // alias for uint64 +// | "uint16" +// | "uint32" +// | "uint64" +// | "uint8" . +// +// Named instances of the boolean, numeric, and string types are keywords. The +// names are not case sensitive. +// +// Note: The blob type is exchanged between the back end and the API as []byte. +// On 32 bit platforms this limits the size which the implementation can handle +// to 2G. +// +// Boolean types +// +// A boolean type represents the set of Boolean truth values denoted by the +// predeclared constants true and false. The predeclared boolean type is bool. +// +// Duration type +// +// A duration type represents the elapsed time between two instants as an int64 +// nanosecond count. The representation limits the largest representable +// duration to approximately 290 years. +// +// Numeric types +// +// A numeric type represents sets of integer or floating-point values. The +// predeclared architecture-independent numeric types are +// +// uint8 the set of all unsigned 8-bit integers (0 to 255) +// uint16 the set of all unsigned 16-bit integers (0 to 65535) +// uint32 the set of all unsigned 32-bit integers (0 to 4294967295) +// uint64 the set of all unsigned 64-bit integers (0 to 18446744073709551615) +// +// int8 the set of all signed 8-bit integers (-128 to 127) +// int16 the set of all signed 16-bit integers (-32768 to 32767) +// int32 the set of all signed 32-bit integers (-2147483648 to 2147483647) +// int64 the set of all signed 64-bit integers (-9223372036854775808 to 9223372036854775807) +// duration the set of all signed 64-bit integers (-9223372036854775808 to 9223372036854775807) +// bigint the set of all integers +// +// bigrat the set of all rational numbers +// +// float32 the set of all IEEE-754 32-bit floating-point numbers +// float64 the set of all IEEE-754 64-bit floating-point numbers +// +// complex64 the set of all complex numbers with float32 real and imaginary parts +// complex128 the set of all complex numbers with float64 real and imaginary parts +// +// byte alias for uint8 +// float alias for float64 +// int alias for int64 +// rune alias for int32 +// uint alias for uint64 +// +// The value of an n-bit integer is n bits wide and represented using two's +// complement arithmetic. +// +// Conversions are required when different numeric types are mixed in an +// expression or assignment. +// +// String types +// +// A string type represents the set of string values. A string value is a +// (possibly empty) sequence of bytes. The case insensitive keyword for the +// string type is 'string'. +// +// The length of a string (its size in bytes) can be discovered using the +// built-in function len. +// +// Time types +// +// A time type represents an instant in time with nanosecond precision. Each +// time has associated with it a location, consulted when computing the +// presentation form of the time. +// +// Predeclared functions +// +// The following functions are implicitly declared +// +// avg complex contains count date +// day formatTime hasPrefix hasSuffix hour +// hours id imag len max +// min minute minutes month nanosecond +// nanoseconds now parseTime real second +// seconds since sum timeIn weekday +// year yearDay +// +// Expressions +// +// An expression specifies the computation of a value by applying operators and +// functions to operands. +// +// Operands +// +// Operands denote the elementary values in an expression. An operand may be a +// literal, a (possibly qualified) identifier denoting a constant or a function +// or a table/record set column, or a parenthesized expression. +// +// Operand = Literal | QualifiedIdent | "(" Expression ")" . +// Literal = "FALSE" | "NULL" | "TRUE" +// | float_lit | imaginary_lit | int_lit | rune_lit | string_lit +// | ql_parameter . +// +// Qualified identifiers +// +// A qualified identifier is an identifier qualified with a table/record set +// name prefix. +// +// QualifiedIdent = identifier [ "." identifier ] . +// +// For example +// +// invoice.Num // might denote column 'Num' from table 'invoice' +// +// Primary expressions +// +// Primary expression are the operands for unary and binary expressions. +// +// PrimaryExpression = Operand +// | Conversion +// | PrimaryExpression Index +// | PrimaryExpression Slice +// | PrimaryExpression Call . +// +// Call = "(" [ ExpressionList ] ")" . +// Index = "[" Expression "]" . +// Slice = "[" [ Expression ] ":" [ Expression ] "]" . +// +// For example +// +// x +// 2 +// (s + ".txt") +// f(3.1415, true) +// s[i : j + 1] +// +// Index expressions +// +// A primary expression of the form +// +// s[x] +// +// denotes the element of a string indexed by x. Its type is byte. The value x +// is called the index. The following rules apply +// +// - The index x must be of integer type except bigint or duration; it is in +// range if 0 <= x < len(s), otherwise it is out of range. +// +// - A constant index must be non-negative and representable by a value of type +// int. +// +// - A constant index must be in range if the string a is a literal. +// +// - If x is out of range at run time, a run-time error occurs. +// +// - s[x] is the byte at index x and the type of s[x] is byte. +// +// If s is NULL or x is NULL then the result is NULL. +// +// Otherwise s[x] is illegal. +// +// Slices +// +// For a string, the primary expression +// +// s[low : high] +// +// constructs a substring. The indices low and high select which elements +// appear in the result. The result has indices starting at 0 and length equal +// to high - low. +// +// For convenience, any of the indices may be omitted. A missing low index +// defaults to zero; a missing high index defaults to the length of the sliced +// operand +// +// s[2:] // same s[2 : len(s)] +// s[:3] // same as s[0 : 3] +// s[:] // same as s[0 : len(s)] +// +// The indices low and high are in range if 0 <= low <= high <= len(a), +// otherwise they are out of range. A constant index must be non-negative and +// representable by a value of type int. If both indices are constant, they +// must satisfy low <= high. If the indices are out of range at run time, a +// run-time error occurs. +// +// Integer values of type bigint or duration cannot be used as indices. +// +// If s is NULL the result is NULL. If low or high is not omitted and is NULL +// then the result is NULL. +// +// Calls +// +// Given an identifier f denoting a predeclared function, +// +// f(a1, a2, … an) +// +// calls f with arguments a1, a2, … an. Arguments are evaluated before the +// function is called. The type of the expression is the result type of f. +// +// complex(x, y) +// len(name) +// +// In a function call, the function value and arguments are evaluated in the +// usual order. After they are evaluated, the parameters of the call are passed +// by value to the function and the called function begins execution. The +// return value of the function is passed by value when the function returns. +// +// Calling an undefined function causes a compile-time error. +// +// Operators +// +// Operators combine operands into expressions. +// +// Expression = Term { ( oror | "OR" ) Term } . +// +// ExpressionList = Expression { "," Expression } [ "," ]. +// Factor = PrimaryFactor { ( ge | ">" | le | "<" | neq | eq | "LIKE" ) PrimaryFactor } [ Predicate ] . +// PrimaryFactor = PrimaryTerm { ( "^" | "|" | "-" | "+" ) PrimaryTerm } . +// PrimaryTerm = UnaryExpr { ( andnot | "&" | lsh | rsh | "%" | "/" | "*" ) UnaryExpr } . +// Term = Factor { ( andand | "AND" ) Factor } . +// UnaryExpr = [ "^" | "!" | "-" | "+" ] PrimaryExpression . +// +// Comparisons are discussed elsewhere. For other binary operators, the operand +// types must be identical unless the operation involves shifts or untyped +// constants. For operations involving constants only, see the section on +// constant expressions. +// +// Except for shift operations, if one operand is an untyped constant and the +// other operand is not, the constant is converted to the type of the other +// operand. +// +// The right operand in a shift expression must have unsigned integer type or +// be an untyped constant that can be converted to unsigned integer type. If +// the left operand of a non-constant shift expression is an untyped constant, +// the type of the constant is what it would be if the shift expression were +// replaced by its left operand alone. +// +// Pattern matching +// +// Expressions of the form +// +// expr1 LIKE expr2 +// +// yeild a boolean value true if expr2, a regular expression, matches expr1 +// (see also [6]). Both expression must be of type string. If any one of the +// expressions is NULL the result is NULL. +// +// Predicates +// +// Predicates are special form expressions having a boolean result type. +// +// Expressions of the form +// +// expr IN ( expr1, expr2, expr3, ... ) // case A +// +// expr NOT IN ( expr1, expr2, expr3, ... ) // case B +// +// are equivalent, including NULL handling, to +// +// expr == expr1 || expr == expr2 || expr == expr3 || ... // case A +// +// expr != expr1 && expr != expr2 && expr != expr3 && ... // case B +// +// The types of involved expressions must be comparable as defined in +// "Comparison operators". +// +// Another form of the IN predicate creates the expression list from a result +// of a SelectStmt. +// +// DELETE FROM t WHERE id() IN (SELECT id_t FROM u WHERE inactive_days > 365) +// +// The SelectStmt must select only one column. The produced expression list is +// resource limited by the memory available to the process. NULL values +// produced by the SelectStmt are ignored, but if all records of the SelectStmt +// are NULL the predicate yields NULL. The select statement is evaluated only +// once. If the type of expr is not the same as the type of the field returned +// by the SelectStmt then the set operation yields false. The type of the +// column returned by the SelectStmt must be one of the simple (non blob-like) +// types: +// +// bool +// byte // alias uint8 +// complex128 +// complex64 +// float // alias float64 +// float32 +// float64 +// int // alias int64 +// int16 +// int32 +// int64 +// int8 +// rune // alias int32 +// string +// uint // alias uint64 +// uint16 +// uint32 +// uint64 +// uint8 +// +// Expressions of the form +// +// expr BETWEEN low AND high // case A +// +// expr NOT BETWEEN low AND high // case B +// +// are equivalent, including NULL handling, to +// +// expr >= low && expr <= high // case A +// +// expr < low || expr > high // case B +// +// The types of involved expressions must be ordered as defined in "Comparison +// operators". +// +// Predicate = ( +// [ "NOT" ] ( +// "IN" "(" ExpressionList ")" +// | "IN" "(" SelectStmt ")" +// | "BETWEEN" PrimaryFactor "AND" PrimaryFactor +// ) +// | "IS" [ "NOT" ] "NULL" +// ). +// +// Expressions of the form +// +// expr IS NULL // case A +// +// expr IS NOT NULL // case B +// +// yeild a boolean value true if expr does not have a specific type (case A) or +// if expr has a specific type (case B). In other cases the result is a boolean +// value false. +// +// Operator precedence +// +// Unary operators have the highest precedence. +// +// There are five precedence levels for binary operators. Multiplication +// operators bind strongest, followed by addition operators, comparison +// operators, && (logical AND), and finally || (logical OR) +// +// Precedence Operator +// 5 * / % << >> & &^ +// 4 + - | ^ +// 3 == != < <= > >= +// 2 && +// 1 || +// +// Binary operators of the same precedence associate from left to right. For +// instance, x / y * z is the same as (x / y) * z. +// +// +x +// 23 + 3*x[i] +// x <= f() +// ^a >> b +// f() || g() +// x == y+1 && z > 0 +// +// Note that the operator precedence is reflected explicitly by the grammar. +// +// Arithmetic operators +// +// Arithmetic operators apply to numeric values and yield a result of the same +// type as the first operand. The four standard arithmetic operators (+, -, *, +// /) apply to integer, rational, floating-point, and complex types; + also +// applies to strings; +,- also applies to times. All other arithmetic +// operators apply to integers only. +// +// + sum integers, rationals, floats, complex values, strings +// - difference integers, rationals, floats, complex values, times +// * product integers, rationals, floats, complex values +// / quotient integers, rationals, floats, complex values +// % remainder integers +// +// & bitwise AND integers +// | bitwise OR integers +// ^ bitwise XOR integers +// &^ bit clear (AND NOT) integers +// +// << left shift integer << unsigned integer +// >> right shift integer >> unsigned integer +// +// Strings can be concatenated using the + operator +// +// "hi" + string(c) + " and good bye" +// +// String addition creates a new string by concatenating the operands. +// +// A value of type duration can be added to or subtracted from a value of type time. +// +// now() + duration("1h") // time after 1 hour from now +// duration("1h") + now() // time after 1 hour from now +// now() - duration("1h") // time before 1 hour from now +// duration("1h") - now() // illegal, negative times do not exist +// +// Times can subtracted from each other producing a value of type duration. +// +// now() - t0 // elapsed time since t0 +// now() + now() // illegal, operator + not defined for times +// +// For two integer values x and y, the integer quotient q = x / y and remainder +// r = x % y satisfy the following relationships +// +// x = q*y + r and |r| < |y| +// +// with x / y truncated towards zero ("truncated division"). +// +// x y x / y x % y +// 5 3 1 2 +// -5 3 -1 -2 +// 5 -3 -1 2 +// -5 -3 1 -2 +// +// As an exception to this rule, if the dividend x is the most negative value +// for the int type of x, the quotient q = x / -1 is equal to x (and r = 0). +// +// x, q +// int8 -128 +// int16 -32768 +// int32 -2147483648 +// int64 -9223372036854775808 +// +// If the divisor is a constant expression, it must not be zero. If the divisor +// is zero at run time, a run-time error occurs. If the dividend is +// non-negative and the divisor is a constant power of 2, the division may be +// replaced by a right shift, and computing the remainder may be replaced by a +// bitwise AND operation +// +// x x / 4 x % 4 x >> 2 x & 3 +// 11 2 3 2 3 +// -11 -2 -3 -3 1 +// +// The shift operators shift the left operand by the shift count specified by +// the right operand. They implement arithmetic shifts if the left operand is a +// signed integer and logical shifts if it is an unsigned integer. There is no +// upper limit on the shift count. Shifts behave as if the left operand is +// shifted n times by 1 for a shift count of n. As a result, x << 1 is the same +// as x*2 and x >> 1 is the same as x/2 but truncated towards negative +// infinity. +// +// For integer operands, the unary operators +, -, and ^ are defined as follows +// +// +x is 0 + x +// -x negation is 0 - x +// ^x bitwise complement is m ^ x with m = "all bits set to 1" for unsigned x +// and m = -1 for signed x +// +// For floating-point and complex numbers, +x is the same as x, while -x is the +// negation of x. The result of a floating-point or complex division by zero is +// not specified beyond the IEEE-754 standard; whether a run-time error occurs +// is implementation-specific. +// +// Whenever any operand of any arithmetic operation, unary or binary, is NULL, +// as well as in the case of the string concatenating operation, the result is +// NULL. +// +// 42*NULL // the result is NULL +// NULL/x // the result is NULL +// "foo"+NULL // the result is NULL +// +// Integer overflow +// +// For unsigned integer values, the operations +, -, *, and << are computed +// modulo 2n, where n is the bit width of the unsigned integer's type. Loosely +// speaking, these unsigned integer operations discard high bits upon overflow, +// and expressions may rely on ``wrap around''. +// +// For signed integers with a finite bit width, the operations +, -, *, and << +// may legally overflow and the resulting value exists and is deterministically +// defined by the signed integer representation, the operation, and its +// operands. No exception is raised as a result of overflow. An evaluator may +// not optimize an expression under the assumption that overflow does not +// occur. For instance, it may not assume that x < x + 1 is always true. +// +// Integers of type bigint and rationals do not overflow but their handling is +// limited by the memory resources available to the program. +// +// Comparison operators +// +// Comparison operators compare two operands and yield a boolean value. +// +// == equal +// != not equal +// < less +// <= less or equal +// > greater +// >= greater or equal +// +// In any comparison, the first operand must be of same type as is the second +// operand, or vice versa. +// +// The equality operators == and != apply to operands that are comparable. The +// ordering operators <, <=, >, and >= apply to operands that are ordered. +// These terms and the result of the comparisons are defined as follows +// +// - Boolean values are comparable. Two boolean values are equal if they are +// either both true or both false. +// +// - Complex values are comparable. Two complex values u and v are equal if +// both real(u) == real(v) and imag(u) == imag(v). +// +// - Integer values are comparable and ordered, in the usual way. Note that +// durations are integers. +// +// - Floating point values are comparable and ordered, as defined by the +// IEEE-754 standard. +// +// - Rational values are comparable and ordered, in the usual way. +// +// - String values are comparable and ordered, lexically byte-wise. +// +// - Time values are comparable and ordered. +// +// Whenever any operand of any comparison operation is NULL, the result is +// NULL. +// +// Note that slices are always of type string. +// +// Logical operators +// +// Logical operators apply to boolean values and yield a boolean result. The +// right operand is evaluated conditionally. +// +// && conditional AND p && q is "if p then q else false" +// || conditional OR p || q is "if p then true else q" +// ! NOT !p is "not p" +// +// The truth tables for logical operations with NULL values +// +// +-------+-------+---------+---------+ +// | p | q | p || q | p && q | +// +-------+-------+---------+---------+ +// | true | true | *true | true | +// | true | false | *true | false | +// | true | NULL | *true | NULL | +// | false | true | true | *false | +// | false | false | false | *false | +// | false | NULL | NULL | *false | +// | NULL | true | true | NULL | +// | NULL | false | NULL | false | +// | NULL | NULL | NULL | NULL | +// +-------+-------+---------+---------+ +// * indicates q is not evaluated. +// +// +-------+-------+ +// | p | !p | +// +-------+-------+ +// | true | false | +// | false | true | +// | NULL | NULL | +// +-------+-------+ +// +// Conversions +// +// Conversions are expressions of the form T(x) where T is a type and x is an +// expression that can be converted to type T. +// +// Conversion = Type "(" Expression ")" . +// +// A constant value x can be converted to type T in any of these cases: +// +// - x is representable by a value of type T. +// +// - x is a floating-point constant, T is a floating-point type, and x is +// representable by a value of type T after rounding using IEEE 754 +// round-to-even rules. The constant T(x) is the rounded value. +// +// - x is an integer constant and T is a string type. The same rule as for +// non-constant x applies in this case. +// +// Converting a constant yields a typed constant as result. +// +// float32(2.718281828) // 2.718281828 of type float32 +// complex128(1) // 1.0 + 0.0i of type complex128 +// float32(0.49999999) // 0.5 of type float32 +// string('x') // "x" of type string +// string(0x266c) // "♬" of type string +// "foo" + "bar" // "foobar" +// int(1.2) // illegal: 1.2 cannot be represented as an int +// string(65.0) // illegal: 65.0 is not an integer constant +// +// A non-constant value x can be converted to type T in any of these cases: +// +// - x has type T. +// +// - x's type and T are both integer or floating point types. +// +// - x's type and T are both complex types. +// +// - x is an integer, except bigint or duration, and T is a string type. +// +// Specific rules apply to (non-constant) conversions between numeric types or +// to and from a string type. These conversions may change the representation +// of x and incur a run-time cost. All other conversions only change the type +// but not the representation of x. +// +// A conversion of NULL to any type yields NULL. +// +// Conversions between numeric types +// +// For the conversion of non-constant numeric values, the following rules +// apply +// +// 1. When converting between integer types, if the value is a signed integer, +// it is sign extended to implicit infinite precision; otherwise it is zero +// extended. It is then truncated to fit in the result type's size. For +// example, if v == uint16(0x10F0), then uint32(int8(v)) == 0xFFFFFFF0. The +// conversion always yields a valid value; there is no indication of overflow. +// +// 2. When converting a floating-point number to an integer, the fraction is +// discarded (truncation towards zero). +// +// 3. When converting an integer or floating-point number to a floating-point +// type, or a complex number to another complex type, the result value is +// rounded to the precision specified by the destination type. For instance, +// the value of a variable x of type float32 may be stored using additional +// precision beyond that of an IEEE-754 32-bit number, but float32(x) +// represents the result of rounding x's value to 32-bit precision. Similarly, +// x + 0.1 may use more than 32 bits of precision, but float32(x + 0.1) does +// not. +// +// In all non-constant conversions involving floating-point or complex values, +// if the result type cannot represent the value the conversion succeeds but +// the result value is implementation-dependent. +// +// Conversions to and from a string type +// +// 1. Converting a signed or unsigned integer value to a string type yields a +// string containing the UTF-8 representation of the integer. Values outside +// the range of valid Unicode code points are converted to "\uFFFD". +// +// string('a') // "a" +// string(-1) // "\ufffd" == "\xef\xbf\xbd" +// string(0xf8) // "\u00f8" == "ø" == "\xc3\xb8" +// string(0x65e5) // "\u65e5" == "日" == "\xe6\x97\xa5" +// +// 2. Converting a blob to a string type yields a string whose successive bytes +// are the elements of the blob. +// +// string(b /* []byte{'h', 'e', 'l', 'l', '\xc3', '\xb8'} */) // "hellø" +// string(b /* []byte{} */) // "" +// string(b /* []byte(nil) */) // "" +// +// 3. Converting a value of a string type to a blob yields a blob whose +// successive elements are the bytes of the string. +// +// blob("hellø") // []byte{'h', 'e', 'l', 'l', '\xc3', '\xb8'} +// blob("") // []byte{} +// +// 4. Converting a value of a bigint type to a string yields a string +// containing the decimal decimal representation of the integer. +// +// string(M9) // "2305843009213693951" +// +// 5. Converting a value of a string type to a bigint yields a bigint value +// containing the integer represented by the string value. A prefix of “0x” or +// “0X” selects base 16; the “0” prefix selects base 8, and a “0b” or “0B” +// prefix selects base 2. Otherwise the value is interpreted in base 10. An +// error occurs if the string value is not in any valid format. +// +// bigint("2305843009213693951") // M9 +// bigint("0x1ffffffffffffffffffffff") // M10 == 2^89-1 +// +// 6. Converting a value of a rational type to a string yields a string +// containing the decimal decimal representation of the rational in the form +// "a/b" (even if b == 1). +// +// string(bigrat(355)/bigrat(113)) // "355/113" +// +// 7. Converting a value of a string type to a bigrat yields a bigrat value +// containing the rational represented by the string value. The string can be +// given as a fraction "a/b" or as a floating-point number optionally followed +// by an exponent. An error occurs if the string value is not in any valid +// format. +// +// bigrat("1.2e-34") +// bigrat("355/113") +// +// 8. Converting a value of a duration type to a string returns a string +// representing the duration in the form "72h3m0.5s". Leading zero units are +// omitted. As a special case, durations less than one second format using a +// smaller unit (milli-, micro-, or nanoseconds) to ensure that the leading +// digit is non-zero. The zero duration formats as 0, with no unit. +// +// string(elapsed) // "1h", for example +// +// 9. Converting a string value to a duration yields a duration represented by +// the string. A duration string is a possibly signed sequence of decimal +// numbers, each with optional fraction and a unit suffix, such as "300ms", +// "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", +// "m", "h". +// +// duration("1m") // http://golang.org/pkg/time/#Minute +// +// 10. Converting a time value to a string returns the time formatted using the +// format string +// +// "2006-01-02 15:04:05.999999999 -0700 MST" +// +// Order of evaluation +// +// When evaluating the operands of an expression or of function calls, +// operations are evaluated in lexical left-to-right order. +// +// For example, in the evaluation of +// +// g(h(), i()+x[j()], c) +// +// the function calls and evaluation of c happen in the order h(), i(), j(), c. +// +// Floating-point operations within a single expression are evaluated according +// to the associativity of the operators. Explicit parentheses affect the +// evaluation by overriding the default associativity. In the expression x + (y +// + z) the addition y + z is performed before adding x. +// +// Statements +// +// Statements control execution. +// +// Statement = EmptyStmt | AlterTableStmt | BeginTransactionStmt | CommitStmt +// | CreateIndexStmt | CreateTableStmt | DeleteFromStmt | DropIndexStmt +// | DropTableStmt | InsertIntoStmt | RollbackStmt | SelectStmt +// | TruncateTableStmt | UpdateStmt . +// +// StatementList = Statement { ";" Statement } . +// +// Empty statements +// +// The empty statement does nothing. +// +// EmptyStmt = . +// +// ALTER TABLE +// +// Alter table statements modify existing tables. With the ADD clause it adds +// a new column to the table. The column must not exist. With the DROP clause +// it removes an existing column from a table. The column must exist and it +// must be not the only (last) column of the table. IOW, there cannot be a +// table with no columns. +// +// AlterTableStmt = "ALTER" "TABLE" TableName ( "ADD" ColumnDef | "DROP" "COLUMN" ColumnName ) . +// +// For example +// +// BEGIN TRANSACTION; +// ALTER TABLE Stock ADD Qty int; +// ALTER TABLE Income DROP COLUMN Taxes; +// COMMIT; +// +// BEGIN TRANSACTION +// +// Begin transactions statements introduce a new transaction level. Every +// transaction level must be eventually balanced by exactly one of COMMIT or +// ROLLBACK statements. Note that when a transaction is roll-backed because of +// a statement failure then no explicit balancing of the respective BEGIN +// TRANSACTION is statement is required nor permitted. +// +// Failure to properly balance any opened transaction level may cause dead +// locks and/or lose of data updated in the uppermost opened but never properly +// closed transaction level. +// +// BeginTransactionStmt = "BEGIN" "TRANSACTION" . +// +// For example +// +// BEGIN TRANSACTION; +// INSERT INTO foo VALUES (42, 3.14); +// INSERT INTO foo VALUES (-1, 2.78); +// COMMIT; +// +// Mandatory transactions +// +// A database cannot be updated (mutated) outside of a transaction. Statements +// requiring a transaction +// +// ALTER TABLE +// COMMIT +// CREATE INDEX +// CREATE TABLE +// DELETE FROM +// DROP INDEX +// DROP TABLE +// INSERT INTO +// ROLLBACK +// TRUNCATE TABLE +// UPDATE +// +// A database is effectively read only outside of a transaction. Statements not +// requiring a transaction +// +// BEGIN TRANSACTION +// SELECT FROM +// +// COMMIT +// +// The commit statement closes the innermost transaction nesting level. If +// that's the outermost level then the updates to the DB made by the +// transaction are atomically made persistent. +// +// CommitStmt = "COMMIT" . +// +// For example +// +// BEGIN TRANSACTION; +// INSERT INTO AccountA (Amount) VALUES ($1); +// INSERT INTO AccountB (Amount) VALUES (-$1); +// COMMIT; +// +// CREATE INDEX +// +// Create index statements create new indices. Index is a named projection of +// ordered values of a table column to the respective records. As a special +// case the id() of the record can be indexed. Index name must not be the same +// as any of the existing tables and it also cannot be the same as of any +// column name of the table the index is on. +// +// CreateIndexStmt = "CREATE" [ "UNIQUE" ] "INDEX" [ "IF" "NOT" "EXISTS" ] +// IndexName "ON" TableName "(" ( ColumnName | "id" Call ) ")" . +// +// For example +// +// BEGIN TRANSACTION; +// CREATE TABLE Orders (CustomerID int, Date time); +// CREATE INDEX OrdersID ON Orders (id()); +// CREATE INDEX OrdersDate ON Orders (Date); +// CREATE TABLE Items (OrderID int, ProductID int, Qty int); +// CREATE INDEX ItemsOrderID ON Items (OrderID); +// COMMIT; +// +// Now certain SELECT statements may use the indices to speed up joins and/or +// to speed up record set filtering when the WHERE clause is used; or the +// indices might be used to improve the performance when the ORDER BY clause is +// present. +// +// The UNIQUE modifier requires the indexed values to be unique or NULL. +// +// The optional IF NOT EXISTS clause makes the statement a no operation if the +// index already exists. +// +// CREATE TABLE +// +// Create table statements create new tables. A column definition declares the +// column name and type. Table names and column names are case sensitive. +// Neither a table or an index of the same name may exist in the DB. +// +// CreateTableStmt = "CREATE" "TABLE" [ "IF" "NOT" "EXISTS" ] TableName +// "(" ColumnDef { "," ColumnDef } [ "," ] ")" . +// +// ColumnDef = ColumnName Type . +// ColumnName = identifier . +// TableName = identifier . +// +// For example +// +// BEGIN TRANSACTION; +// CREATE TABLE department ( +// DepartmentID int, +// DepartmentName string, +// ); +// CREATE TABLE employee ( +// LastName string, +// DepartmentID int, +// ); +// COMMIT; +// +// The optional IF NOT EXISTS clause makes the statement a no operation if the +// table already exists. +// +// DELETE FROM +// +// Delete from statements remove rows from a table, which must exist. +// +// DeleteFromStmt = "DELETE" "FROM" TableName [ WhereClause ] . +// +// For example +// +// BEGIN TRANSACTION; +// DELETE FROM DepartmentID +// WHERE DepartmentName == "Ponies"; +// COMMIT; +// +// If the WHERE clause is not present then all rows are removed and the +// statement is equivalent to the TRUNCATE TABLE statement. +// +// DROP INDEX +// +// Drop index statements remove indices from the DB. The index must exist. +// +// DropIndexStmt = "DROP" "INDEX" [ "IF" "EXISTS" ] IndexName . +// IndexName = identifier . +// +// For example +// +// BEGIN TRANSACTION; +// DROP INDEX ItemsOrderID; +// COMMIT; +// +// The optional IF EXISTS clause makes the statement a no operation if the +// index does not exist. +// +// DROP TABLE +// +// Drop table statements remove tables from the DB. The table must exist. +// +// DropTableStmt = "DROP" "TABLE" [ "IF" "EXISTS" ] TableName . +// +// For example +// +// BEGIN TRANSACTION; +// DROP TABLE Inventory; +// COMMIT; +// +// The optional IF EXISTS clause makes the statement a no operation if the +// table does not exist. +// +// INSERT INTO +// +// Insert into statements insert new rows into tables. New rows come from +// literal data, if using the VALUES clause, or are a result of select +// statement. In the later case the select statement is fully evaluated before +// the insertion of any rows is performed, allowing to insert values calculated +// from the same table rows are to be inserted into. If the ColumnNameList part +// is omitted then the number of values inserted in the row must be the same as +// are columns in the table. If the ColumnNameList part is present then the +// number of values per row must be same as the same number of column names. +// All other columns of the record are set to NULL. The type of the value +// assigned to a column must be the same as is the column's type or the value +// must be NULL. +// +// InsertIntoStmt = "INSERT" "INTO" TableName [ "(" ColumnNameList ")" ] ( Values | SelectStmt ) . +// +// ColumnNameList = ColumnName { "," ColumnName } [ "," ] . +// Values = "VALUES" "(" ExpressionList ")" { "," "(" ExpressionList ")" } [ "," ] . +// +// For example +// +// BEGIN TRANSACTION; +// INSERT INTO department (DepartmentID) VALUES (42); +// +// INSERT INTO department ( +// DepartmentName, +// DepartmentID, +// ) +// VALUES ( +// "R&D", +// 42, +// ); +// +// INSERT INTO department VALUES +// (42, "R&D"), +// (17, "Sales"), +// ; +// COMMIT; +// +// BEGIN TRANSACTION; +// INSERT INTO department (DepartmentName, DepartmentID) +// SELECT DepartmentName+"/headquarters", DepartmentID+1000 +// FROM department; +// COMMIT; +// +// ROLLBACK +// +// The rollback statement closes the innermost transaction nesting level +// discarding any updates to the DB made by it. If that's the outermost level +// then the effects on the DB are as if the transaction never happened. +// +// RollbackStmt = "ROLLBACK" . +// +// For example +// +// // First statement list +// BEGIN TRANSACTION +// SELECT * INTO tmp FROM foo; +// INSERT INTO tmp SELECT * from bar; +// SELECT * from tmp; +// +// The (temporary) record set from the last statement is returned and can be +// processed by the client. +// +// // Second statement list +// ROLLBACK; +// +// In this case the rollback is the same as 'DROP TABLE tmp;' but it can be a +// more complex operation. +// +// SELECT FROM +// +// Select from statements produce recordsets. The optional DISTINCT modifier +// ensures all rows in the result recordset are unique. Either all of the +// resulting fields are returned ('*') or only those named in FieldList. +// +// RecordSetList is a list of table names or parenthesized select statements, +// optionally (re)named using the AS clause. +// +// The result can be filtered using a WhereClause and orderd by the OrderBy +// clause. +// +// SelectStmt = "SELECT" [ "DISTINCT" ] ( "*" | FieldList ) "FROM" RecordSetList +// [ WhereClause ] [ GroupByClause ] [ OrderBy ] [ Limit ] [ Offset ]. +// +// RecordSet = ( TableName | "(" SelectStmt [ ";" ] ")" ) [ "AS" identifier ] . +// RecordSetList = RecordSet { "," RecordSet } [ "," ] . +// +// For example +// +// SELECT * FROM Stock; +// +// SELECT DepartmentID +// FROM department +// WHERE DepartmentID == 42 +// ORDER BY DepartmentName; +// +// SELECT employee.LastName +// FROM department, employee +// WHERE department.DepartmentID == employee.DepartmentID +// ORDER BY DepartmentID; +// +// If Recordset is a nested, parenthesized SelectStmt then it must be given a +// name using the AS clause if its field are to be accessible in expressions. +// +// SELECT a.b, c.d +// FROM +// x AS a, +// ( +// SELECT * FROM y; +// ) AS c +// WHERE a.e > c.e; +// +// Fields naming rules +// +// A field is an named expression. Identifiers, not used as a type in +// conversion or a function name in the Call clause, denote names of (other) +// fields, values of which should be used in the expression. +// +// Field = Expression [ "AS" identifier ] . +// +// The expression can be named using the AS clause. If the AS clause is not +// present and the expression consists solely of a field name, then that field +// name is used as the name of the resulting field. Otherwise the field is +// unnamed. +// +// For example +// +// SELECT 314, 42 as AUQLUE, DepartmentID, DepartmentID+1000, LastName as Name from employee; +// // Fields are []string{"", "AUQLUE", "DepartmentID", "", "Name"} +// +// The SELECT statement can optionally enumerate the desired/resulting fields +// in a list. +// +// FieldList = Field { "," Field } [ "," ] . +// +// No two identical field names can appear in the list. +// +// SELECT DepartmentID, LastName, DepartmentID from employee; +// // duplicate field name "DepartmentID" +// +// SELECT DepartmentID, LastName, DepartmentID as ID2 from employee; +// // works +// +// When more than one record set is used in the FROM clause record set list, +// the result record set field names are rewritten to be qualified using +// the record set names. +// +// SELECT * FROM employee, department; +// // Fields are []string{"employee.LastName", "employee.DepartmentID", "department.DepartmentID", "department.DepartmentName" +// +// If a particular record set doesn't have a name, its respective fields became +// unnamed. +// +// SELECT * FROM employee as e, ( SELECT * FROM department); +// // Fields are []string{"e.LastName", "e.DepartmentID", "", "" +// +// SELECT * FROM employee AS e, ( SELECT * FROM department) AS d; +// // Fields are []string{"e.LastName", "e.DepartmentID", "d.DepartmentID", "d.DepartmentName" +// +// Recordset ordering +// +// Resultins rows of a SELECT statement can be optionally ordered by the ORDER +// BY clause. Collating proceeds by considering the expressions in the +// expression list left to right until a collating order is determined. Any +// possibly remaining expressions are not evaluated. +// +// OrderBy = "ORDER" "BY" ExpressionList [ "ASC" | "DESC" ] . +// +// All of the expression values must yield an ordered type or NULL. Ordered +// types are defined in "Comparison operators". Collating of elements having a +// NULL value is different compared to what the comparison operators yield in +// expression evaluation (NULL result instead of a boolean value). +// +// Below, T denotes a non NULL value of any QL type. +// +// NULL < T +// +// NULL collates before any non NULL value (is considered smaller than T). +// +// NULL == NULL +// +// Two NULLs have no collating order (are considered equal). +// +// Recordset filtering +// +// The WHERE clause restricts records considered by some statements, like +// SELECT FROM, DELETE FROM, or UPDATE. +// +// expression value consider the record +// ---------------- ------------------- +// true yes +// false or NULL no +// +// It is an error if the expression evaluates to a non null value of non bool +// type. +// +// WhereClause = "WHERE" Expression . +// +// Recordset grouping +// +// The GROUP BY clause is used to project rows having common values into a +// smaller set of rows. +// +// For example +// +// SELECT Country, sum(Qty) FROM Sales GROUP BY Country; +// +// SELECT Country, Product FROM Sales GROUP BY Country, Product; +// +// SELECT DISTINCT Country, Product FROM Sales; +// +// Using the GROUP BY without any aggregate functions in the selected fields is +// in certain cases equal to using the DISTINCT modifier. The last two examples +// above produce the same resultsets. +// +// GroupByClause = "GROUP BY" ColumnNameList . +// +// Skipping records +// +// The optional OFFSET clause allows to ignore first N records. For example +// +// SELECT * FROM t OFFSET 10; +// +// The above will produce only rows 11, 12, ... of the record set, if they +// exist. The value of the expression must a non negative integer, but not +// bigint or duration. +// +// Offset = "OFFSET" Expression . +// +// Limiting the result set size +// +// The optional LIMIT clause allows to ignore all but first N records. For +// example +// +// SELECT * FROM t LIMIT 10; +// +// The above will return at most the first 10 records of the record set. The +// value of the expression must a non negative integer, but not bigint or +// duration. +// +// Limit = "Limit" Expression . +// +// The LIMIT and OFFSET clauses can be combined. For example +// +// SELECT * FROM t LIMIT 5 OFFSET 3; +// +// Considering table t has, say 10 records, the above will produce only records +// 4 - 8. +// +// #1: Ignore 1/3 +// #2: Ignore 2/3 +// #3: Ignore 3/3 +// #4: Return 1/5 +// #5: Return 2/5 +// #6: Return 3/5 +// #7: Return 4/5 +// #8: Return 5/5 +// +// After returning record #8, no more result rows/records are computed. +// +// Select statement evaluation order +// +// 1. The FROM clause is evaluated, producing a Cartesian product of its source +// record sets (tables or nested SELECT statements). +// +// 2. If present, the WHERE clause is evaluated on the result set of the +// previous evaluation. +// +// 3. If present, the GROUP BY clause is evaluated on the result set of the +// previous evaluation(s). +// +// 4. The SELECT field expressions are evaluated on the result set of the +// previous evaluation(s). +// +// 5. If present, the DISTINCT modifier is evaluated on the result set of the +// previous evaluation(s). +// +// 6. If present, the ORDER BY clause is evaluated on the result set of the +// previous evaluation(s). +// +// 7. If present, the OFFSET clause is evaluated on the result set of the +// previous evaluation(s). The offset expression is evaluated once for the +// first record produced by the previous evaluations. +// +// 8. If present, the LIMIT clause is evaluated on the result set of the +// previous evaluation(s). The limit expression is evaluated once for the first +// record produced by the previous evaluations. +// +// +// TRUNCATE TABLE +// +// Truncate table statements remove all records from a table. The table must +// exist. +// +// TruncateTableStmt = "TRUNCATE" "TABLE" TableName . +// +// For example +// +// BEGIN TRANSACTION +// TRUNCATE TABLE department; +// COMMIT; +// +// UPDATE +// +// Update statements change values of fields in rows of a table. +// +// UpdateStmt = "UPDATE" TableName [ "SET" ] AssignmentList [ WhereClause ] . +// +// AssignmentList = Assignment { "," Assignment } [ "," ] . +// Assignment = ColumnName "=" Expression . +// +// For example +// +// BEGIN TRANSACTION +// UPDATE department +// DepartmentName = DepartmentName + " dpt.", +// DepartmentID = 1000+DepartmentID, +// WHERE DepartmentID < 1000; +// COMMIT; +// +// Note: The SET clause is optional. +// +// System Tables +// +// To allow to query for DB meta data, there exist specially named virtual +// tables. +// +// Note: System tables have fake table-wise unique but meaningless and unstable +// record IDs. Do not apply the built-in id() to any system table. +// +// Tables Table +// +// The table __Table lists all tables in the DB. The schema is +// +// CREATE TABLE __Table (Name string, Schema string); +// +// The Schema column returns the statement to (re)create table Name. +// +// Columns Table +// +// The table __Colum lists all columns of all tables in the DB. The schema is +// +// CREATE TABLE __Column (TableName string, Ordinal int, Name string, Type string); +// +// The Ordinal column defines the 1-based index of the column in the record. +// +// Indices table +// +// The table __Index lists all indices in the DB. The schema is +// +// CREATE TABLE __Index (TableName string, ColumnName string, Name string, IsUnique bool); +// +// The IsUnique columns reflects if the index was created using the optional +// UNIQUE clause. +// +// Built-in functions +// +// Built-in functions are predeclared. +// +// Average +// +// The built-in aggregate function avg returns the average of values of an +// expression. Avg ignores NULL values, but returns NULL if all values of a +// column are NULL or if avg is applied to an empty record set. +// +// func avg(e numeric) typeof(e) +// +// The column values must be of a numeric type. +// +// SELECT salesperson, avg(sales) FROM salesforce GROUP BY salesperson; +// +// Contains +// +// The built-in function contains returns true if substr is within s. +// +// func contains(s, substr string) bool +// +// If any argument to contains is NULL the result is NULL. +// +// Count +// +// The built-in aggregate function count returns how many times an expression +// has a non NULL values or the number of rows in a record set. Note: count() +// returns 0 for an empty record set. +// +// func count() int // The number of rows in a record set. +// func count(e expression) int // The number of cases where the expression value is not NULL. +// +// For example +// +// SELECT count() FROM department; // # of rows +// +// SELECT count(DepartmentID) FROM department; // # of records with non NULL field DepartmentID +// +// SELECT count()-count(DepartmentID) FROM department; // # of records with NULL field DepartmentID +// +// SELECT count(foo+bar*3) AS y FROM t; // # of cases where 'foo+bar*3' is non NULL +// +// Date +// +// Date returns the time corresponding to +// +// yyyy-mm-dd hh:mm:ss + nsec nanoseconds +// +// in the appropriate zone for that time in the given location. +// +// The month, day, hour, min, sec, and nsec values may be outside their usual +// ranges and will be normalized during the conversion. For example, October 32 +// converts to November 1. +// +// A daylight savings time transition skips or repeats times. For example, in +// the United States, March 13, 2011 2:15am never occurred, while November 6, +// 2011 1:15am occurred twice. In such cases, the choice of time zone, and +// therefore the time, is not well-defined. Date returns a time that is correct +// in one of the two zones involved in the transition, but it does not +// guarantee which. +// +// func date(year, month, day, hour, min, sec, nsec int, loc string) time +// +// A location maps time instants to the zone in use at that time. Typically, +// the location represents the collection of time offsets in use in a +// geographical area, such as "CEST" and "CET" for central Europe. "local" +// represents the system's local time zone. "UTC" represents Universal +// Coordinated Time (UTC). +// +// The month specifies a month of the year (January = 1, ...). +// +// If any argument to date is NULL the result is NULL. +// +// Day +// +// The built-in function day returns the day of the month specified by t. +// +// func day(t time) int +// +// If the argument to day is NULL the result is NULL. +// +// Format time +// +// The built-in function formatTime returns a textual representation of the +// time value formatted according to layout, which defines the format by +// showing how the reference time, +// +// Mon Jan 2 15:04:05 -0700 MST 2006 +// +// would be displayed if it were the value; it serves as an example of the +// desired output. The same display rules will then be applied to the time +// value. +// +// func formatTime(t time, layout string) string +// +// If any argument to formatTime is NULL the result is NULL. +// +// NOTE: The string value of the time zone, like "CET" or "ACDT", is dependent +// on the time zone of the machine the function is run on. For example, if the +// t value is in "CET", but the machine is in "ACDT", instead of "CET" the +// result is "+0100". This is the same what Go (time.Time).String() returns and +// in fact formatTime directly calls t.String(). +// +// formatTime(date(2006, 1, 2, 15, 4, 5, 999999999, "CET")) +// +// returns +// +// 2006-01-02 15:04:05.999999999 +0100 CET +// +// on a machine in the CET time zone, but may return +// +// 2006-01-02 15:04:05.999999999 +0100 +0100 +// +// on a machine in the ACDT zone. The time value is in both cases the same so +// its ordering and comparing is correct. Only the display value can differ. +// +// HasPrefix +// +// The built-in function hasPrefix tests whether the string s begins with prefix. +// +// func hasPrefix(s, prefix string) bool +// +// If any argument to hasPrefix is NULL the result is NULL. +// +// HasSuffix +// +// The built-in function hasSuffix tests whether the string s ends with suffix. +// +// func hasSuffix(s, suffix string) bool +// +// If any argument to hasSuffix is NULL the result is NULL. +// +// Hour +// +// The built-in function hour returns the hour within the day specified by t, +// in the range [0, 23]. +// +// func hour(t time) int +// +// If the argument to hour is NULL the result is NULL. +// +// Hours +// +// The built-in function hours returns the duration as a floating point number +// of hours. +// +// func hours(d duration) float +// +// If the argument to hours is NULL the result is NULL. +// +// Record id +// +// The built-in function id takes zero or one arguments. If no argument is +// provided, id() returns a table-unique automatically assigned numeric +// identifier of type int. Ids of deleted records are not reused unless the DB +// becomes completely empty (has no tables). +// +// func id() int +// +// For example +// +// SELECT id(), LastName +// FROM employee; +// +// If id() without arguments is called for a row which is not a table record +// then the result value is NULL. +// +// For example +// +// SELECT id(), e.LastName, e.DepartmentID, d.DepartmentID +// FROM +// employee AS e, +// department AS d, +// WHERE e.DepartmentID == d.DepartmentID; +// // Will always return NULL in first field. +// +// SELECT e.ID, e.LastName, e.DepartmentID, d.DepartmentID +// FROM +// (SELECT id() AS ID, LastName, DepartmentID FROM employee) AS e, +// department as d, +// WHERE e.DepartmentID == d.DepartmentID; +// // Will work. +// +// If id() has one argument it must be a table name of a table in a cross join. +// +// For example +// +// SELECT * +// FROM foo, bar +// WHERE bar.fooID == id(foo) +// ORDER BY id(foo); +// +// Length +// +// The built-in function len takes a string argument and returns the lentgh of +// the string in bytes. +// +// func len(s string) int +// +// The expression len(s) is constant if s is a string constant. +// +// If the argument to len is NULL the result is NULL. +// +// Maximum +// +// The built-in aggregate function max returns the largest value of an +// expression in a record set. Max ignores NULL values, but returns NULL if +// all values of a column are NULL or if max is applied to an empty record set. +// +// func max(e expression) typeof(e) // The largest value of the expression. +// +// The expression values must be of an ordered type. +// +// For example +// +// SELECT department, max(sales) FROM t GROUP BY department; +// +// Minimum +// +// The built-in aggregate function min returns the smallest value of an +// expression in a record set. Min ignores NULL values, but returns NULL if +// all values of a column are NULL or if min is applied to an empty record set. +// +// func min(e expression) typeof(e) // The smallest value of the expression. +// +// For example +// +// SELECT a, min(b) FROM t GROUP BY a; +// +// The column values must be of an ordered type. +// +// Minute +// +// The built-in function minute returns the minute offset within the hour +// specified by t, in the range [0, 59]. +// +// func minute(t time) int +// +// If the argument to minute is NULL the result is NULL. +// +// Minutes +// +// The built-in function minutes returns the duration as a floating point +// number of minutes. +// +// func minutes(d duration) float +// +// If the argument to minutes is NULL the result is NULL. +// +// Month +// +// The built-in function month returns the month of the year specified by t +// (January = 1, ...). +// +// func month(t time) int +// +// If the argument to month is NULL the result is NULL. +// +// Nanosecond +// +// The built-in function nanosecond returns the nanosecond offset within the +// second specified by t, in the range [0, 999999999]. +// +// func nanosecond(t time) int +// +// If the argument to nanosecond is NULL the result is NULL. +// +// Nanoseconds +// +// The built-in function nanoseconds returns the duration as an integer +// nanosecond count. +// +// func nanoseconds(d duration) float +// +// If the argument to nanoseconds is NULL the result is NULL. +// +// Now +// +// The built-in function now returns the current local time. +// +// func now() time +// +// Parse time +// +// The built-in function parseTime parses a formatted string and returns the +// time value it represents. The layout defines the format by showing how the +// reference time, +// +// Mon Jan 2 15:04:05 -0700 MST 2006 +// +// would be interpreted if it were the value; it serves as an example of the +// input format. The same interpretation will then be made to the input string. +// +// Elements omitted from the value are assumed to be zero or, when zero is +// impossible, one, so parsing "3:04pm" returns the time corresponding to Jan +// 1, year 0, 15:04:00 UTC (note that because the year is 0, this time is +// before the zero Time). Years must be in the range 0000..9999. The day of the +// week is checked for syntax but it is otherwise ignored. +// +// In the absence of a time zone indicator, parseTime returns a time in UTC. +// +// When parsing a time with a zone offset like -0700, if the offset corresponds +// to a time zone used by the current location, then parseTime uses that +// location and zone in the returned time. Otherwise it records the time as +// being in a fabricated location with time fixed at the given zone offset. +// +// When parsing a time with a zone abbreviation like MST, if the zone +// abbreviation has a defined offset in the current location, then that offset +// is used. The zone abbreviation "UTC" is recognized as UTC regardless of +// location. If the zone abbreviation is unknown, Parse records the time as +// being in a fabricated location with the given zone abbreviation and a zero +// offset. This choice means that such a time can be parses and reformatted +// with the same layout losslessly, but the exact instant used in the +// representation will differ by the actual zone offset. To avoid such +// problems, prefer time layouts that use a numeric zone offset. +// +// func parseTime(layout, value string) time +// +// If any argument to parseTime is NULL the result is NULL. +// +// Second +// +// The built-in function second returns the second offset within the minute +// specified by t, in the range [0, 59]. +// +// func second(t time) int +// +// If the argument to second is NULL the result is NULL. +// +// Seconds +// +// The built-in function seconds returns the duration as a floating point +// number of seconds. +// +// func seconds(d duration) float +// +// If the argument to seconds is NULL the result is NULL. +// +// Since +// +// The built-in function since returns the time elapsed since t. It is +// shorthand for now()-t. +// +// func since(t time) duration +// +// If the argument to since is NULL the result is NULL. +// +// Sum +// +// The built-in aggregate function sum returns the sum of values of an +// expression for all rows of a record set. Sum ignores NULL values, but +// returns NULL if all values of a column are NULL or if sum is applied to an +// empty record set. +// +// func sum(e expression) typeof(e) // The sum of the values of the expression. +// +// The column values must be of a numeric type. +// +// SELECT salesperson, sum(sales) FROM salesforce GROUP BY salesperson; +// +// Time in a specific zone +// +// The built-in function timeIn returns t with the location information set to +// loc. For discussion of the loc argument please see date(). +// +// func timeIn(t time, loc string) time +// +// If any argument to timeIn is NULL the result is NULL. +// +// Weekday +// +// The built-in function weekday returns the day of the week specified by t. +// Sunday == 0, Monday == 1, ... +// +// func weekday(t time) int +// +// If the argument to weekday is NULL the result is NULL. +// +// Year +// +// The built-in function year returns the year in which t occurs. +// +// func year(t time) int +// +// If the argument to year is NULL the result is NULL. +// +// Year day +// +// The built-in function yearDay returns the day of the year specified by t, in +// the range [1,365] for non-leap years, and [1,366] in leap years. +// +// func yearDay(t time) int +// +// If the argument to yearDay is NULL the result is NULL. +// +// Manipulating complex numbers +// +// Three functions assemble and disassemble complex numbers. The built-in +// function complex constructs a complex value from a floating-point real and +// imaginary part, while real and imag extract the real and imaginary parts of +// a complex value. +// +// complex(realPart, imaginaryPart floatT) complexT +// real(complexT) floatT +// imag(complexT) floatT +// +// The type of the arguments and return value correspond. For complex, the two +// arguments must be of the same floating-point type and the return type is the +// complex type with the corresponding floating-point constituents: complex64 +// for float32, complex128 for float64. The real and imag functions together +// form the inverse, so for a complex value z, z == complex(real(z), imag(z)). +// +// If the operands of these functions are all constants, the return value is a +// constant. +// +// complex(2, -2) // complex128 +// complex(1.0, -1.4) // complex128 +// float32(math.Cos(math.Pi/2)) // float32 +// complex(5, float32(-x)) // complex64 +// imag(b) // float64 +// real(complex(5, float32(-x))) // float32 +// +// If any argument to any of complex, real, imag functions is NULL the result +// is NULL. +// +// Size guarantees +// +// For the numeric types, the following sizes are guaranteed +// +// type size in bytes +// +// byte, uint8, int8 1 +// uint16, int16 2 +// uint32, int32, float32 4 +// uint, uint64, int, int64, float64, complex64 8 +// complex128 16 +// +// License +// +// Portions of this specification page are modifications based on work[2] +// created and shared by Google[3] and used according to terms described in the +// Creative Commons 3.0 Attribution License[4]. +// +// This specification is licensed under the Creative Commons Attribution 3.0 +// License, and code is licensed under a BSD license[5]. +// +// References +// +// Links from the above documentation +// +// [1]: http://golang.org/ref/spec#Notation +// [2]: http://golang.org/ref/spec +// [3]: http://code.google.com/policies.html +// [4]: http://creativecommons.org/licenses/by/3.0/ +// [5]: http://golang.org/LICENSE +// [6]: http://golang.org/pkg/regexp/#Regexp.MatchString +// [7]: http://developer.mimer.com/validator/sql-reserved-words.tml +// +// Implementation details +// +// This section is not part of the specification. +// +// Indices +// +// WARNING: The implementation of indices is new and it surely needs more time +// to become mature. +// +// Indices are used currently used only by the WHERE clause. The following +// expression patterns of 'WHERE expression' are recognized and trigger index +// use. +// +// - WHERE c // For bool typed indexed column c +// - WHERE !c // For bool typed indexed column c +// - WHERE c relOp constExpr // For indexed column c +// - WHERE c relOp parameter // For indexed column c +// - WHERE parameter relOp c // For indexed column c +// - WHERE constExpr relOp c // For indexed column c +// +// The relOp is one of the relation operators <, <=, ==, >=, >. For the +// equality operator both operands must be of comparable types. For all other +// operators both operands must be of ordered types. The constant expression is +// a compile time constant expression. Some constant folding is still a TODO. +// Parameter is a QL parameter ($1 etc.). +// +// Query rewriting +// +// Consider tables t and u, both with an indexed field f. The WHERE expression +// doesn't comply with the above simple detected cases. +// +// SELECT * FROM t, u WHERE t.f < x && u.f < y; +// +// However, such query is now automatically rewritten to +// +// SELECT * FROM +// (SELECT * FROM t WHERE f < x), +// (SELECT * FROM u WHERE f < y); +// +// which will use both of the indices. The impact of using the indices can be +// substantial (cf. BenchmarkCrossJoin*) if the resulting rows have low +// "selectivity", ie. only few rows from both tables are selected by the +// respective WHERE filtering. +// +// Note: Existing QL DBs can be used and indices can be added to them. However, +// once any indices are present in the DB, the old QL versions cannot work with +// such DB anymore. +// +// Benchmarks +// +// Running a benchmark with -v (-test.v) outputs information about the scale +// used to report records/s and a brief description of the benchmark. For +// example +// +// $ go test -run NONE -bench 'SelectMem.*1e[23]' -v +// PASS +// BenchmarkSelectMem1kBx1e2 50000 67680 ns/op 1477537.05 MB/s +// --- BENCH: BenchmarkSelectMem1kBx1e2 +// all_test.go:310: +// ============================================================= +// NOTE: All benchmarks report records/s as 1000000 bytes/s. +// ============================================================= +// all_test.go:321: Having a table of 100 records, each of size 1kB, measure the performance of +// SELECT * FROM t; +// +// BenchmarkSelectMem1kBx1e3 5000 634819 ns/op 1575251.01 MB/s +// --- BENCH: BenchmarkSelectMem1kBx1e3 +// all_test.go:321: Having a table of 1000 records, each of size 1kB, measure the performance of +// SELECT * FROM t; +// +// ok github.com/cznic/ql 7.496s +// $ +// +// Running the full suite of benchmarks takes a lot of time. Use the -timeout +// flag to avoid them being killed after the default time limit (10 minutes). +package ql diff --git a/Godeps/_workspace/src/github.com/cznic/ql/driver.go b/Godeps/_workspace/src/github.com/cznic/ql/driver.go new file mode 100644 index 00000000000..7aa69b4c19e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/driver.go @@ -0,0 +1,528 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// database/sql/driver + +package ql + +import ( + "bytes" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "math/big" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ( + _ driver.Conn = (*driverConn)(nil) + _ driver.Driver = (*sqlDriver)(nil) + _ driver.Execer = (*driverConn)(nil) + _ driver.Queryer = (*driverConn)(nil) + _ driver.Result = (*driverResult)(nil) + _ driver.Rows = (*driverRows)(nil) + _ driver.Stmt = (*driverStmt)(nil) + _ driver.Tx = (*driverConn)(nil) + + txBegin = MustCompile("BEGIN TRANSACTION;") + txCommit = MustCompile("COMMIT;") + txRollback = MustCompile("ROLLBACK;") + + errNoResult = errors.New("query statement does not produce a result set (no top level SELECT)") +) + +type errList []error + +func (e *errList) append(err error) { + if err != nil { + *e = append(*e, err) + } +} + +func (e errList) error() error { + if len(e) == 0 { + return nil + } + + return e +} + +func (e errList) Error() string { + a := make([]string, len(e)) + for i, v := range e { + a[i] = v.Error() + } + return strings.Join(a, "\n") +} + +func params(args []driver.Value) []interface{} { + r := make([]interface{}, len(args)) + for i, v := range args { + r[i] = interface{}(v) + } + return r +} + +var ( + fileDriver = &sqlDriver{dbs: map[string]*driverDB{}} + fileDriverOnce sync.Once + memDriver = &sqlDriver{isMem: true, dbs: map[string]*driverDB{}} + memDriverOnce sync.Once +) + +// RegisterDriver registers a QL database/sql/driver[0] named "ql". The name +// parameter of +// +// sql.Open("ql", name) +// +// is interpreted as a path name to a named DB file which will be created if +// not present. The underlying QL database data are persisted on db.Close(). +// RegisterDriver can be safely called multiple times, it'll register the +// driver only once. +// +// The name argument can be optionally prefixed by "file://". In that case the +// prefix is stripped before interpreting it as a file name. +// +// The name argument can be optionally prefixed by "memory://". In that case +// the prefix is stripped before interpreting it as a name of a memory-only, +// volatile DB. +// +// [0]: http://golang.org/pkg/database/sql/driver/ +func RegisterDriver() { + fileDriverOnce.Do(func() { sql.Register("ql", fileDriver) }) +} + +// RegisterMemDriver registers a QL memory database/sql/driver[0] named +// "ql-mem". The name parameter of +// +// sql.Open("ql-mem", name) +// +// is interpreted as an unique memory DB name which will be created if not +// present. The underlying QL memory database data are not persisted on +// db.Close(). RegisterMemDriver can be safely called multiple times, it'll +// register the driver only once. +// +// [0]: http://golang.org/pkg/database/sql/driver/ +func RegisterMemDriver() { + memDriverOnce.Do(func() { sql.Register("ql-mem", memDriver) }) +} + +type driverDB struct { + db *DB + name string + refcount int +} + +func newDriverDB(db *DB, name string) *driverDB { + return &driverDB{db: db, name: name, refcount: 1} +} + +// sqlDriver implements the interface required by database/sql/driver. +type sqlDriver struct { + dbs map[string]*driverDB + isMem bool + mu sync.Mutex +} + +func (d *sqlDriver) lock() func() { + d.mu.Lock() + return d.mu.Unlock +} + +// Open returns a new connection to the database. The name is a string in a +// driver-specific format. +// +// Open may return a cached connection (one previously closed), but doing so is +// unnecessary; the sql package maintains a pool of idle connections for +// efficient re-use. +// +// The returned connection is only used by one goroutine at a time. +func (d *sqlDriver) Open(name string) (driver.Conn, error) { + if d != fileDriver && d != memDriver { + return nil, fmt.Errorf("open: unexpected/unsupported instance of driver.Driver: %p", d) + } + + switch { + case d == fileDriver && strings.HasPrefix(name, "file://"): + name = name[len("file://"):] + case d == fileDriver && strings.HasPrefix(name, "memory://"): + d = memDriver + name = name[len("memory://"):] + } + name = filepath.Clean(name) + if name == "" || name == "." || name == string(os.PathSeparator) { + return nil, fmt.Errorf("invalid DB name %q", name) + } + + defer d.lock()() + db := d.dbs[name] + if db == nil { + var err error + var db0 *DB + switch d.isMem { + case true: + db0, err = OpenMem() + default: + db0, err = OpenFile(name, &Options{CanCreate: true}) + } + if err != nil { + return nil, err + } + + db = newDriverDB(db0, name) + d.dbs[name] = db + return newDriverConn(d, db), nil + } + + db.refcount++ + return newDriverConn(d, db), nil +} + +// driverConn is a connection to a database. It is not used concurrently by +// multiple goroutines. +// +// Conn is assumed to be stateful. +type driverConn struct { + ctx *TCtx + db *driverDB + driver *sqlDriver + stop map[*driverStmt]struct{} + tnl int +} + +func newDriverConn(d *sqlDriver, ddb *driverDB) driver.Conn { + r := &driverConn{ + db: ddb, + driver: d, + stop: map[*driverStmt]struct{}{}, + } + return r +} + +// Prepare returns a prepared statement, bound to this connection. +func (c *driverConn) Prepare(query string) (driver.Stmt, error) { + list, err := Compile(query) + if err != nil { + return nil, err + } + + s := &driverStmt{conn: c, stmt: list} + c.stop[s] = struct{}{} + return s, nil +} + +// Close invalidates and potentially stops any current prepared statements and +// transactions, marking this connection as no longer in use. +// +// Because the sql package maintains a free pool of connections and only calls +// Close when there's a surplus of idle connections, it shouldn't be necessary +// for drivers to do their own connection caching. +func (c *driverConn) Close() error { + var err errList + for s := range c.stop { + err.append(s.Close()) + } + defer c.driver.lock()() + dbs, name := c.driver.dbs, c.db.name + v := dbs[name] + v.refcount-- + if v.refcount == 0 { + err.append(c.db.db.Close()) + delete(dbs, name) + } + return err.error() +} + +// Begin starts and returns a new transaction. +func (c *driverConn) Begin() (driver.Tx, error) { + if c.ctx == nil { + c.ctx = NewRWCtx() + } + + if _, _, err := c.db.db.Execute(c.ctx, txBegin); err != nil { + return nil, err + } + + c.tnl++ + return c, nil +} + +func (c *driverConn) Commit() error { + if c.tnl == 0 || c.ctx == nil { + return errCommitNotInTransaction + } + + if _, _, err := c.db.db.Execute(c.ctx, txCommit); err != nil { + return err + } + + c.tnl-- + if c.tnl == 0 { + c.ctx = nil + } + return nil +} + +func (c *driverConn) Rollback() error { + if c.tnl == 0 || c.ctx == nil { + return errRollbackNotInTransaction + } + + if _, _, err := c.db.db.Execute(c.ctx, txRollback); err != nil { + return err + } + + c.tnl-- + if c.tnl == 0 { + c.ctx = nil + } + return nil +} + +// Execer is an optional interface that may be implemented by a Conn. +// +// If a Conn does not implement Execer, the sql package's DB.Exec will first +// prepare a query, execute the statement, and then close the statement. +// +// Exec may return driver.ErrSkip. +func (c *driverConn) Exec(query string, args []driver.Value) (driver.Result, error) { + list, err := Compile(query) + if err != nil { + return nil, err + } + + return driverExec(c.db, c.ctx, list, args) +} + +func driverExec(db *driverDB, ctx *TCtx, list List, args []driver.Value) (driver.Result, error) { + if _, _, err := db.db.Execute(ctx, list, params(args)...); err != nil { + return nil, err + } + + if len(list.l) == 1 { + switch list.l[0].(type) { + case *createTableStmt, *dropTableStmt, *alterTableAddStmt, + *alterTableDropColumnStmt, *truncateTableStmt: + return driver.ResultNoRows, nil + } + } + + r := &driverResult{} + if ctx != nil { + r.lastInsertID, r.rowsAffected = ctx.LastInsertID, ctx.RowsAffected + } + return r, nil +} + +// Queryer is an optional interface that may be implemented by a Conn. +// +// If a Conn does not implement Queryer, the sql package's DB.Query will first +// prepare a query, execute the statement, and then close the statement. +// +// Query may return driver.ErrSkip. +func (c *driverConn) Query(query string, args []driver.Value) (driver.Rows, error) { + list, err := Compile(query) + if err != nil { + return nil, err + } + + return driverQuery(c.db, c.ctx, list, args) +} + +func driverQuery(db *driverDB, ctx *TCtx, list List, args []driver.Value) (driver.Rows, error) { + rss, _, err := db.db.Execute(ctx, list, params(args)...) + if err != nil { + return nil, err + } + + switch n := len(rss); n { + case 0: + return nil, errNoResult + case 1: + rs := rss[n-1] + if x, ok := rss[n-1].(recordset); ok { + x.tx = ctx + rs = x + } + return newdriverRows(rs), nil + default: + return nil, fmt.Errorf("query produced %d result sets, expected only one", n) + } +} + +// driverResult is the result of a query execution. +type driverResult struct { + lastInsertID int64 + rowsAffected int64 +} + +// LastInsertId returns the database's auto-generated ID after, for example, an +// INSERT into a table with primary key. +func (r *driverResult) LastInsertId() (int64, error) { // -golint + return r.lastInsertID, nil +} + +// RowsAffected returns the number of rows affected by the query. +func (r *driverResult) RowsAffected() (int64, error) { + return r.rowsAffected, nil +} + +// driverRows is an iterator over an executed query's results. +type driverRows struct { + rs Recordset + done chan int + rows chan interface{} +} + +func newdriverRows(rs Recordset) *driverRows { + r := &driverRows{ + rs: rs, + done: make(chan int), + rows: make(chan interface{}, 500), + } + go func() { + err := io.EOF + if e := r.rs.Do(false, func(data []interface{}) (bool, error) { + select { + case r.rows <- data: + return true, nil + case <-r.done: + return false, nil + } + }); e != nil { + err = e + } + + select { + case r.rows <- err: + case <-r.done: + } + }() + return r +} + +// Columns returns the names of the columns. The number of columns of the +// result is inferred from the length of the slice. If a particular column +// name isn't known, an empty string should be returned for that entry. +func (r *driverRows) Columns() []string { + f, _ := r.rs.Fields() + return f +} + +// Close closes the rows iterator. +func (r *driverRows) Close() error { + close(r.done) + return nil +} + +// Next is called to populate the next row of data into the provided slice. The +// provided slice will be the same size as the Columns() are wide. +// +// The dest slice may be populated only with a driver Value type, but excluding +// string. All string values must be converted to []byte. +// +// Next should return io.EOF when there are no more rows. +func (r *driverRows) Next(dest []driver.Value) error { + select { + case rx := <-r.rows: + switch x := rx.(type) { + case error: + return x + case []interface{}: + if g, e := len(x), len(dest); g != e { + return fmt.Errorf("field count mismatch: got %d, need %d", g, e) + } + + for i, xi := range x { + switch v := xi.(type) { + case nil, int64, float64, bool, []byte, time.Time: + dest[i] = v + case complex64, complex128, *big.Int, *big.Rat: + var buf bytes.Buffer + fmt.Fprintf(&buf, "%v", v) + dest[i] = buf.Bytes() + case int8: + dest[i] = int64(v) + case int16: + dest[i] = int64(v) + case int32: + dest[i] = int64(v) + case int: + dest[i] = int64(v) + case uint8: + dest[i] = int64(v) + case uint16: + dest[i] = int64(v) + case uint32: + dest[i] = int64(v) + case uint64: + dest[i] = int64(v) + case uint: + dest[i] = int64(v) + case time.Duration: + dest[i] = int64(v) + case string: + dest[i] = []byte(v) + default: + return fmt.Errorf("internal error 004") + } + } + return nil + default: + return fmt.Errorf("internal error 005") + } + case <-r.done: + return io.EOF + } +} + +// driverStmt is a prepared statement. It is bound to a driverConn and not used +// by multiple goroutines concurrently. +type driverStmt struct { + conn *driverConn + stmt List +} + +// Close closes the statement. +// +// As of Go 1.1, a Stmt will not be closed if it's in use by any queries. +func (s *driverStmt) Close() error { + delete(s.conn.stop, s) + return nil +} + +// NumInput returns the number of placeholder parameters. +// +// If NumInput returns >= 0, the sql package will sanity check argument counts +// from callers and return errors to the caller before the statement's Exec or +// Query methods are called. +// +// NumInput may also return -1, if the driver doesn't know its number of +// placeholders. In that case, the sql package will not sanity check Exec or +// Query argument counts. +func (s *driverStmt) NumInput() int { + if x := s.stmt; len(x.l) == 1 { + return x.params + } + + return -1 +} + +// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE. +func (s *driverStmt) Exec(args []driver.Value) (driver.Result, error) { + c := s.conn + return driverExec(c.db, c.ctx, s.stmt, args) +} + +// Exec executes a query that may return rows, such as a SELECT. +func (s *driverStmt) Query(args []driver.Value) (driver.Rows, error) { + c := s.conn + return driverQuery(c.db, c.ctx, s.stmt, args) +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/driver/Makefile b/Godeps/_workspace/src/github.com/cznic/ql/driver/Makefile new file mode 100644 index 00000000000..f1b64ee6630 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/driver/Makefile @@ -0,0 +1,40 @@ +# Copyright (c) 2014 The ql Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +.PHONY: all clean nuke + +all: editor + go install + go vet + golint . + make todo + +bench: all + go test -run NONE -bench . + +clean: + go clean + rm -f *~ + +cover: + t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t + +editor: + go fmt + go test -i + go test + go build + +nuke: + go clean -i + +todo: + @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go || true + @grep -n TODO *.go || true + @grep -n BUG *.go || true + @grep -n println *.go || true + +later: + @grep -n LATER *.go || true + @grep -n MAYBE *.go || true diff --git a/Godeps/_workspace/src/github.com/cznic/ql/driver/all_test.go b/Godeps/_workspace/src/github.com/cznic/ql/driver/all_test.go new file mode 100644 index 00000000000..db28d1dedbd --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/driver/all_test.go @@ -0,0 +1,202 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package driver + +import ( + "database/sql" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +func Example_testFile() { + dir, err := ioutil.TempDir("", "ql-driver-test") + if err != nil { + return + } + + defer func() { + os.RemoveAll(dir) + }() + + db, err := sql.Open("ql", filepath.Join(dir, "ql.db")) + if err != nil { + return + } + + defer func() { + if err := db.Close(); err != nil { + return + } + + fmt.Println("OK") + }() + + tx, err := db.Begin() + if err != nil { + return + } + + if _, err := tx.Exec("CREATE TABLE t (Qty int, Name string);"); err != nil { + return + } + + result, err := tx.Exec(` + INSERT INTO t VALUES + ($1, $2), + ($3, $4), + ; + `, + 42, "foo", + 314, "bar", + ) + if err != nil { + return + } + + if err = tx.Commit(); err != nil { + return + } + + id, err := result.LastInsertId() + if err != nil { + return + } + + aff, err := result.RowsAffected() + if err != nil { + return + } + + fmt.Printf("LastInsertId %d, RowsAffected %d\n", id, aff) + + rows, err := db.Query("SELECT * FROM t;") + if err != nil { + return + } + + cols, err := rows.Columns() + if err != nil { + return + } + + fmt.Printf("Columns: %v\n", cols) + + var data struct { + Qty int + Name string + } + + for rows.Next() { + if err = rows.Scan(&data.Qty, &data.Name); err != nil { + rows.Close() + break + } + + fmt.Printf("%+v\n", data) + } + + if err = rows.Err(); err != nil { + return + } + + // Output: + // LastInsertId 2, RowsAffected 2 + // Columns: [Qty Name] + // {Qty:314 Name:bar} + // {Qty:42 Name:foo} + // OK +} + +func Example_testMem() { + db, err := sql.Open("ql-mem", "mem.db") + if err != nil { + return + } + + defer func() { + if err := db.Close(); err != nil { + return + } + + fmt.Println("OK") + }() + + tx, err := db.Begin() + if err != nil { + return + } + + if _, err := tx.Exec("CREATE TABLE t (Qty int, Name string);"); err != nil { + return + } + + result, err := tx.Exec(` + INSERT INTO t VALUES + ($1, $2), + ($3, $4), + ; + `, + 1042, "foo-mem", + 1314, "bar-mem", + ) + if err != nil { + return + } + + if err = tx.Commit(); err != nil { + return + } + + id, err := result.LastInsertId() + if err != nil { + return + } + + aff, err := result.RowsAffected() + if err != nil { + return + } + + fmt.Printf("LastInsertId %d, RowsAffected %d\n", id, aff) + + rows, err := db.Query("SELECT * FROM t;") + if err != nil { + return + } + + cols, err := rows.Columns() + if err != nil { + return + } + + fmt.Printf("Columns: %v\n", cols) + + var data struct { + Qty int + Name string + } + + for rows.Next() { + if err = rows.Scan(&data.Qty, &data.Name); err != nil { + rows.Close() + break + } + + fmt.Printf("%+v\n", data) + } + + if err = rows.Err(); err != nil { + return + } + + // Output: + // LastInsertId 2, RowsAffected 2 + // Columns: [Qty Name] + // {Qty:1314 Name:bar-mem} + // {Qty:1042 Name:foo-mem} + // OK +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/driver/driver.go b/Godeps/_workspace/src/github.com/cznic/ql/driver/driver.go new file mode 100644 index 00000000000..557c70d8286 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/driver/driver.go @@ -0,0 +1,61 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package driver registers a QL sql/driver named "ql" and a memory driver named "ql-mem". + +See also [0], [1] and [3]. + +Usage + +A skeleton program using ql/driver. + + package main + + import ( + "database/sql" + + _ "github.com/cznic/ql/driver" + ) + + func main() { + ... + // Disk file DB + db, err := sql.Open("ql", "ql.db") // [2] + // alternatively + db, err := sql.Open("ql", "file://ql.db") + + // and/or + + // RAM DB + mdb, err := sql.Open("ql-mem", "mem.db") + // alternatively + mdb, err := sql.Open("ql", "memory://mem.db") + if err != nil { + log.Fatal(err) + } + + // Use db/mdb here + ... + } + +This package exports nothing. + +Links + +Referenced from above: + + [0]: http://godoc.org/github.com/cznic/ql + [1]: http://golang.org/pkg/database/sql/ + [2]: http://golang.org/pkg/database/sql/#Open + [3]: http://golang.org/pkg/database/sql/driver +*/ +package driver + +import "github.com/cznic/ql" + +func init() { + ql.RegisterDriver() + ql.RegisterMemDriver() +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/errors.go b/Godeps/_workspace/src/github.com/cznic/ql/errors.go new file mode 100644 index 00000000000..305414f4a9d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/errors.go @@ -0,0 +1,18 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "errors" +) + +var ( + errBeginTransNoCtx = errors.New("BEGIN TRANSACTION: Must use R/W context, have nil") + errCommitNotInTransaction = errors.New("COMMIT: Not in transaction") + errDivByZero = errors.New("division by zero") + errIncompatibleDBFormat = errors.New("incompatible DB format") + errNoDataForHandle = errors.New("read: no data for handle") + errRollbackNotInTransaction = errors.New("ROLLBACK: Not in transaction") +) diff --git a/Godeps/_workspace/src/github.com/cznic/ql/etc.go b/Godeps/_workspace/src/github.com/cznic/ql/etc.go new file mode 100644 index 00000000000..7ac74bb497a --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/etc.go @@ -0,0 +1,2798 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "fmt" + "io" + "log" + "math" + "math/big" + "strings" + "time" +) + +// QL types. +const ( + qBool = 0x62 // 'b' + qComplex64 = 0x63 // 'c' + qComplex128 = 0x64 // 'd' + qFloat32 = 0x66 // 'f' + qFloat64 = 0x67 // 'g', alias float + qInt8 = 0x69 // 'i' + qInt16 = 0x6a // 'j' + qInt32 = 0x6b // 'k' + qInt64 = 0x6c // 'l', alias int + qString = 0x73 // 's' + qUint8 = 0x75 // 'u', alias byte + qUint16 = 0x76 // 'v' + qUint32 = 0x77 // 'w' + qUint64 = 0x78 // 'x', alias uint + + qBigInt = 0x49 // 'I' + qBigRat = 0x52 // 'R' + qBlob = 0x42 // 'B' + qDuration = 0x44 // 'D' + qTime = 0x54 // 'T' +) + +var ( + type2Str = map[int]string{ + qBigInt: "bigint", + qBigRat: "bigrat", + qBlob: "blob", + qBool: "bool", + qComplex128: "complex128", + qComplex64: "complex64", + qDuration: "duration", + qFloat32: "float32", + qFloat64: "float64", + qInt16: "int16", + qInt32: "int32", + qInt64: "int64", + qInt8: "int8", + qString: "string", + qTime: "time", + qUint16: "uint16", + qUint32: "uint32", + qUint64: "uint64", + qUint8: "uint8", + } +) + +func typeStr(typ int) (r string) { + return type2Str[typ] +} + +func noEOF(err error) error { + if err == io.EOF { + err = nil + } + return err +} + +func runErr(err error) error { return fmt.Errorf("run time error: %s", err) } + +func invXOp(s, x interface{}) error { + return fmt.Errorf("invalid operation: %v[%v] (index of type %T)", s, x, x) +} + +func invSOp(s interface{}) error { + return fmt.Errorf("cannot slice %s (type %T)", s, s) +} + +func invNegX(x interface{}) error { + return fmt.Errorf("invalid string index %v (index must be non-negative)", x) +} + +func invNegLO(x interface{}) error { + return fmt.Errorf("invalid LIMIT or OFFSET value %v (must be non-negative)", x) +} + +func invSliceNegX(x interface{}) error { + return fmt.Errorf("invalid slice index %v (index must be non-negative)", x) +} + +func invBoundX(s string, x uint64) error { + return fmt.Errorf("invalid string index %d (out of bounds for %d-byte string)", x, len(s)) +} + +func invSliceBoundX(s string, x uint64) error { + return fmt.Errorf("invalid slice index %d (out of bounds for %d-byte string)", x, len(s)) +} + +func intExpr(x interface{}) (i int64, err error) { + switch x := x.(type) { + case idealInt: + if x < 0 { + return 0, invNegLO(x) + } + + return int64(x), nil + case idealRune: + if x < 0 { + return 0, invNegLO(x) + } + + return int64(x), nil + case idealUint: + if x < 0 { + return 0, invNegLO(x) + } + + return int64(x), nil + case int8: + if x < 0 { + return 0, invNegLO(x) + } + + return int64(x), nil + case int16: + if x < 0 { + return 0, invNegLO(x) + } + + return int64(x), nil + case int32: + if x < 0 { + return 0, invNegLO(x) + } + + return int64(x), nil + case int64: + if x < 0 { + return 0, invNegLO(x) + } + + return int64(x), nil + case uint8: + return int64(x), nil + case uint16: + return int64(x), nil + case uint32: + return int64(x), nil + case uint64: + return int64(x), nil + default: + return 0, fmt.Errorf("non-integer expression: %v (value of type %T)", x, x) + } +} + +func limOffExpr(x interface{}) (i uint64, err error) { + switch x := x.(type) { + case idealInt: + if x < 0 { + return 0, invNegLO(x) + } + + return uint64(x), nil + case idealRune: + if x < 0 { + return 0, invNegLO(x) + } + + return uint64(x), nil + case idealUint: + if x < 0 { + return 0, invNegLO(x) + } + + return uint64(x), nil + case int8: + if x < 0 { + return 0, invNegLO(x) + } + + return uint64(x), nil + case int16: + if x < 0 { + return 0, invNegLO(x) + } + + return uint64(x), nil + case int32: + if x < 0 { + return 0, invNegLO(x) + } + + return uint64(x), nil + case int64: + if x < 0 { + return 0, invNegLO(x) + } + + return uint64(x), nil + case uint8: + return uint64(x), nil + case uint16: + return uint64(x), nil + case uint32: + return uint64(x), nil + case uint64: + return uint64(x), nil + default: + return 0, fmt.Errorf("non-integer used in LIMIT or OFFSET: %v (value of type %T)", x, x) + } +} + +func indexExpr(s *string, x interface{}) (i uint64, err error) { + switch x := x.(type) { + case idealFloat: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && int(x) >= len(*s) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case idealInt: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && int64(x) >= int64(len(*s)) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case idealRune: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && int32(x) >= int32(len(*s)) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case idealUint: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && uint64(x) >= uint64(len(*s)) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int8: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && int(x) >= len(*s) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int16: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && int(x) >= len(*s) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int32: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && int(x) >= len(*s) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int64: + if x < 0 { + return 0, invNegX(x) + } + + if s != nil && x >= int64(len(*s)) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint8: + if s != nil && int(x) >= len(*s) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint16: + if s != nil && int(x) >= len(*s) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint32: + if s != nil && x >= uint32(len(*s)) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint64: + if s != nil && x >= uint64(len(*s)) { + return 0, invBoundX(*s, uint64(x)) + } + + return uint64(x), nil + default: + return 0, fmt.Errorf("non-integer string index %v", x) + } +} + +func sliceExpr(s *string, x interface{}, mod int) (i uint64, err error) { + switch x := x.(type) { + case idealFloat: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && int(x) >= len(*s)+mod { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case idealInt: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && int64(x) >= int64(len(*s)+mod) { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case idealRune: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && int32(x) >= int32(len(*s)+mod) { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case idealUint: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && uint64(x) >= uint64(len(*s)+mod) { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int8: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && int(x) >= len(*s)+mod { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int16: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && int(x) >= len(*s)+mod { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int32: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && int(x) >= len(*s)+mod { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case int64: + if x < 0 { + return 0, invSliceNegX(x) + } + + if s != nil && x >= int64(len(*s)+mod) { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint8: + if s != nil && int(x) >= len(*s)+mod { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint16: + if s != nil && int(x) >= len(*s)+mod { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint32: + if s != nil && x >= uint32(len(*s)+mod) { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + case uint64: + if s != nil && x >= uint64(len(*s)+mod) { + return 0, invSliceBoundX(*s, uint64(x)) + } + + return uint64(x), nil + default: + return 0, fmt.Errorf("invalid slice index %s (type %T)", x, x) + } +} + +type iop int + +func (o iop) String() string { + switch i := int(o); i { + case andand: + return "&&" + case andnot: + return "&^" + case lsh: + return "<<" + case le: + return "<=" + case eq: + return "==" + case ge: + return ">=" + case neq: + return "!=" + case oror: + return "||" + case rsh: + return ">>" + default: + return string(i) + } +} + +func ideal(v interface{}) interface{} { + switch x := v.(type) { + case idealComplex: + return complex128(x) + case idealFloat: + return float64(x) + case idealInt: + return int64(x) + case idealRune: + return int64(x) + case idealUint: + return uint64(x) + default: + return v + } +} + +func eval(v expression, execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (y interface{}) { + y, err := expand1(v.eval(execCtx, ctx, arg)) + if err != nil { + panic(err) // panic ok here + } + return +} + +func eval2(a, b expression, execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (x, y interface{}) { + return eval(a, execCtx, ctx, arg), eval(b, execCtx, ctx, arg) +} + +func invOp2(x, y interface{}, o int) (interface{}, error) { + return nil, fmt.Errorf("invalid operation: %v %v %v (mismatched types %T and %T)", x, iop(o), y, ideal(x), ideal(y)) +} + +func undOp(x interface{}, o int) (interface{}, error) { + return nil, fmt.Errorf("invalid operation: %v%v (operator %v not defined on %T)", iop(o), x, iop(o), x) +} + +func undOp2(x, y interface{}, o int) (interface{}, error) { + return nil, fmt.Errorf("invalid operation: %v %v %v (operator %v not defined on %T)", x, iop(o), y, iop(o), x) +} + +func invConv(val interface{}, typ int) (interface{}, error) { + return nil, fmt.Errorf("cannot convert %v (type %T) to type %s", val, val, typeStr(typ)) +} + +func truncConv(val interface{}) (interface{}, error) { + return nil, fmt.Errorf("constant %v truncated to integer", val) +} + +func convert(val interface{}, typ int) (v interface{}, err error) { //NTYPE + if val == nil { + return nil, nil + } + + switch typ { + case qBool: + switch x := val.(type) { + //case nil: + //case idealComplex: + //case idealFloat: + //case idealInt: + //case idealRune: + //case idealUint: + case bool: + return bool(x), nil + //case complex64: + //case complex128: + //case float32: + //case float64: + //case int8: + //case int16: + //case int32: + //case int64: + //case string: + //case uint8: + //case uint16: + //case uint32: + //case uint64: + default: + return invConv(val, typ) + } + case qComplex64: + switch x := val.(type) { + //case nil: + case idealComplex: + return complex64(x), nil + case idealFloat: + return complex(float32(x), 0), nil + case idealInt: + return complex(float32(x), 0), nil + case idealRune: + return complex(float32(x), 0), nil + case idealUint: + return complex(float32(x), 0), nil + //case bool: + case complex64: + return complex64(x), nil + case complex128: + return complex64(x), nil + //case float32: + //case float64: + //case int8: + //case int16: + //case int32: + //case int64: + //case string: + //case uint8: + //case uint16: + //case uint32: + //case uint64: + default: + return invConv(val, typ) + } + case qComplex128: + switch x := val.(type) { + //case nil: + case idealComplex: + return complex128(x), nil + case idealFloat: + return complex(float64(x), 0), nil + case idealInt: + return complex(float64(x), 0), nil + case idealRune: + return complex(float64(x), 0), nil + case idealUint: + return complex(float64(x), 0), nil + //case bool: + case complex64: + return complex128(x), nil + case complex128: + return complex128(x), nil + //case float32: + //case float64: + //case int8: + //case int16: + //case int32: + //case int64: + //case string: + //case uint8: + //case uint16: + //case uint32: + //case uint64: + default: + return invConv(val, typ) + } + case qFloat32: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + return float32(x), nil + case idealInt: + return float32(x), nil + case idealRune: + return float32(x), nil + case idealUint: + return float32(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return float32(x), nil + case float64: + return float32(x), nil + case int8: + return float32(x), nil + case int16: + return float32(x), nil + case int32: + return float32(x), nil + case int64: + return float32(x), nil + //case string: + case uint8: + return float32(x), nil + case uint16: + return float32(x), nil + case uint32: + return float32(x), nil + case uint64: + return float32(x), nil + case *big.Int: + v, _ := big.NewRat(1, 1).SetInt(x).Float64() + return float32(v), nil + case *big.Rat: + v, _ := x.Float64() + return float32(v), nil + case time.Duration: + return float32(x), nil + default: + return invConv(val, typ) + } + case qFloat64: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + return float64(x), nil + case idealInt: + return float64(x), nil + case idealRune: + return float64(x), nil + case idealUint: + return float64(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return float64(x), nil + case float64: + return float64(x), nil + case int8: + return float64(x), nil + case int16: + return float64(x), nil + case int32: + return float64(x), nil + case int64: + return float64(x), nil + //case string: + case uint8: + return float64(x), nil + case uint16: + return float64(x), nil + case uint32: + return float64(x), nil + case uint64: + return float64(x), nil + case *big.Int: + v, _ := big.NewRat(1, 1).SetInt(x).Float64() + return v, nil + case *big.Rat: + v, _ := x.Float64() + return v, nil + case time.Duration: + return float64(x), nil + default: + return invConv(val, typ) + } + case qInt8: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return int8(x), nil + case idealInt: + return int8(x), nil + case idealRune: + return int8(x), nil + case idealUint: + return int8(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return int8(x), nil + case float64: + return int8(x), nil + case int8: + return int8(x), nil + case int16: + return int8(x), nil + case int32: + return int8(x), nil + case int64: + return int8(x), nil + //case string: + case uint8: + return int8(x), nil + case uint16: + return int8(x), nil + case uint32: + return int8(x), nil + case uint64: + return int8(x), nil + case *big.Int: + return int8(x.Int64()), nil + case time.Duration: + return int8(x), nil + default: + return invConv(val, typ) + } + case qInt16: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return int16(x), nil + case idealInt: + return int16(x), nil + case idealRune: + return int16(x), nil + case idealUint: + return int16(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return int16(x), nil + case float64: + return int16(x), nil + case int8: + return int16(x), nil + case int16: + return int16(x), nil + case int32: + return int16(x), nil + case int64: + return int16(x), nil + //case string: + case uint8: + return int16(x), nil + case uint16: + return int16(x), nil + case uint32: + return int16(x), nil + case uint64: + return int16(x), nil + case *big.Int: + return int16(x.Int64()), nil + case time.Duration: + return int16(x), nil + default: + return invConv(val, typ) + } + case qInt32: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return int32(x), nil + case idealInt: + return int32(x), nil + case idealRune: + return int32(x), nil + case idealUint: + return int32(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return int32(x), nil + case float64: + return int32(x), nil + case int8: + return int32(x), nil + case int16: + return int32(x), nil + case int32: + return int32(x), nil + case int64: + return int32(x), nil + //case string: + case uint8: + return int32(x), nil + case uint16: + return int32(x), nil + case uint32: + return int32(x), nil + case uint64: + return int32(x), nil + case *big.Int: + return int32(x.Int64()), nil + case time.Duration: + return int32(x), nil + default: + return invConv(val, typ) + } + case qInt64: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return int64(x), nil + case idealInt: + return int64(x), nil + case idealRune: + return int64(x), nil + case idealUint: + return int64(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return int64(x), nil + case float64: + return int64(x), nil + case int8: + return int64(x), nil + case int16: + return int64(x), nil + case int32: + return int64(x), nil + case int64: + return int64(x), nil + //case string: + case uint8: + return int64(x), nil + case uint16: + return int64(x), nil + case uint32: + return int64(x), nil + case uint64: + return int64(x), nil + case *big.Int: + return x.Int64(), nil + case time.Duration: + return int64(x), nil + default: + return invConv(val, typ) + } + case qString: + switch x := val.(type) { + //case nil: + //case idealComplex: + //case idealFloat: + case idealInt: + return string(x), nil + case idealRune: + return string(x), nil + case idealUint: + return string(x), nil + //case bool: + //case complex64: + //case complex128: + //case float32: + //case float64: + case int8: + return string(x), nil + case int16: + return string(x), nil + case int32: + return string(x), nil + case int64: + return string(x), nil + case string: + return string(x), nil + case uint8: + return string(x), nil + case uint16: + return string(x), nil + case uint32: + return string(x), nil + case uint64: + return string(x), nil + case []byte: + return string(x), nil + case *big.Int: + return x.String(), nil + case time.Time: + return x.String(), nil + case time.Duration: + return x.String(), nil + default: + return invConv(val, typ) + } + case qUint8: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return uint8(x), nil + case idealInt: + return uint8(x), nil + case idealRune: + return uint8(x), nil + case idealUint: + return uint8(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return uint8(x), nil + case float64: + return uint8(x), nil + case int8: + return uint8(x), nil + case int16: + return uint8(x), nil + case int32: + return uint8(x), nil + case int64: + return uint8(x), nil + //case string: + case uint8: + return uint8(x), nil + case uint16: + return uint8(x), nil + case uint32: + return uint8(x), nil + case uint64: + return uint8(x), nil + case *big.Int: + return uint8(x.Int64()), nil + case time.Duration: + return uint8(x), nil + default: + return invConv(val, typ) + } + case qUint16: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return uint16(x), nil + case idealInt: + return uint16(x), nil + case idealRune: + return uint16(x), nil + case idealUint: + return uint16(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return uint16(x), nil + case float64: + return uint16(x), nil + case int8: + return uint16(x), nil + case int16: + return uint16(x), nil + case int32: + return uint16(x), nil + case int64: + return uint16(x), nil + //case string: + case uint8: + return uint16(x), nil + case uint16: + return uint16(x), nil + case uint32: + return uint16(x), nil + case uint64: + return uint16(x), nil + case *big.Int: + return uint16(x.Int64()), nil + case time.Duration: + return uint16(x), nil + default: + return invConv(val, typ) + } + case qUint32: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return uint32(x), nil + case idealInt: + return uint32(x), nil + case idealRune: + return uint32(x), nil + case idealUint: + return uint32(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return uint32(x), nil + case float64: + return uint32(x), nil + case int8: + return uint32(x), nil + case int16: + return uint32(x), nil + case int32: + return uint32(x), nil + case int64: + return uint32(x), nil + //case string: + case uint8: + return uint32(x), nil + case uint16: + return uint32(x), nil + case uint32: + return uint32(x), nil + case uint64: + return uint32(x), nil + case *big.Int: + return uint32(x.Int64()), nil + case time.Duration: + return uint32(x), nil + default: + return invConv(val, typ) + } + case qUint64: + switch x := val.(type) { + //case nil: + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + return uint64(x), nil + case idealInt: + return uint64(x), nil + case idealRune: + return uint64(x), nil + case idealUint: + return uint64(x), nil + //case bool: + //case complex64: + //case complex128: + case float32: + return uint64(x), nil + case float64: + return uint64(x), nil + case int8: + return uint64(x), nil + case int16: + return uint64(x), nil + case int32: + return uint64(x), nil + case int64: + return uint64(x), nil + //case string: + case uint8: + return uint64(x), nil + case uint16: + return uint64(x), nil + case uint32: + return uint64(x), nil + case uint64: + return uint64(x), nil + case *big.Int: + return x.Uint64(), nil + case time.Duration: + return uint64(x), nil + default: + return invConv(val, typ) + } + case qBlob: + switch x := val.(type) { + case string: + return []byte(x), nil + case []byte: + return x, nil + default: + return invConv(val, typ) + } + case qBigInt: + switch x := val.(type) { + // case blob + // case bool + //case idealComplex: + case idealFloat: + if _, frac := math.Modf(float64(x)); frac != 0 { + return truncConv(x) + } + + rr := big.NewRat(1, 1).SetFloat64(float64(x)) + ii := big.NewInt(0).Set(rr.Num()) + ii.Quo(ii, rr.Denom()) + return ii, nil + case idealInt: + return big.NewInt(0).SetInt64(int64(x)), nil + case idealRune: + return big.NewInt(0).SetInt64(int64(x)), nil + case idealUint: + return big.NewInt(0).SetUint64(uint64(x)), nil + //case complex64 + //case complex128 + case float32: + rr := big.NewRat(1, 1).SetFloat64(float64(x)) + ii := big.NewInt(0).Set(rr.Num()) + ii.Quo(ii, rr.Denom()) + return ii, nil + case float64: + rr := big.NewRat(1, 1).SetFloat64(float64(x)) + ii := big.NewInt(0).Set(rr.Num()) + ii.Quo(ii, rr.Denom()) + return ii, nil + case int8: + return big.NewInt(0).SetInt64(int64(x)), nil + case int16: + return big.NewInt(0).SetInt64(int64(x)), nil + case int32: + return big.NewInt(0).SetInt64(int64(x)), nil + case int64: + return big.NewInt(0).SetInt64(x), nil + case string: + y := big.NewInt(0) + if _, ok := y.SetString(x, 0); !ok { + return invConv(val, typ) + } + + return y, nil + case uint8: + return big.NewInt(0).SetUint64(uint64(x)), nil + case uint16: + return big.NewInt(0).SetUint64(uint64(x)), nil + case uint32: + return big.NewInt(0).SetUint64(uint64(x)), nil + case uint64: + return big.NewInt(0).SetUint64(x), nil + case *big.Int: + return x, nil + case *big.Rat: + ii := big.NewInt(0).Set(x.Num()) + ii.Div(ii, x.Denom()) + return ii, nil + default: + return invConv(val, typ) + } + case qBigRat: + switch x := val.(type) { + // case blob + // case bool + //case idealComplex: + case idealFloat: + return big.NewRat(1, 1).SetFloat64(float64(x)), nil + case idealInt: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case idealRune: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case idealUint: + return big.NewRat(1, 1).SetInt(big.NewInt(0).SetUint64(uint64(x))), nil + //case complex64 + //case complex128 + case float32: + return big.NewRat(1, 1).SetFloat64(float64(x)), nil + case float64: + return big.NewRat(1, 1).SetFloat64(x), nil + case int8: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case int16: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case int32: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case int64: + return big.NewRat(1, 1).SetInt64(x), nil + case string: + y := big.NewRat(1, 1) + if _, ok := y.SetString(x); !ok { + return invConv(val, typ) + } + + return y, nil + case uint8: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case uint16: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case uint32: + return big.NewRat(1, 1).SetInt64(int64(x)), nil + case uint64: + return big.NewRat(1, 1).SetInt(big.NewInt(0).SetUint64(x)), nil + case *big.Int: + return big.NewRat(1, 1).SetInt(x), nil + case *big.Rat: + return x, nil + default: + return invConv(val, typ) + } + case qDuration: + switch x := val.(type) { + // case blob + // case bool + //case idealComplex: + case idealFloat: + return time.Duration(x), nil + case idealInt: + return time.Duration(x), nil + case idealRune: + return time.Duration(x), nil + case idealUint: + return time.Duration(x), nil + //case complex64 + //case complex128 + case float32: + return time.Duration(x), nil + case float64: + return time.Duration(x), nil + case int8: + return time.Duration(x), nil + case int16: + return time.Duration(x), nil + case int32: + return time.Duration(x), nil + case int64: + return time.Duration(x), nil + case string: + return time.ParseDuration(x) + case uint8: + return time.Duration(x), nil + case uint16: + return time.Duration(x), nil + case uint32: + return time.Duration(x), nil + case uint64: + return time.Duration(x), nil + case *big.Int: + return time.Duration(x.Int64()), nil + case *big.Rat: + f, _ := x.Float64() + return time.Duration(f), nil + case time.Duration: + return x, nil + default: + return invConv(val, typ) + } + case qTime: + switch x := val.(type) { + // case blob + // case bool + //case idealComplex: + //case idealFloat: + //case idealInt: + //case idealRune: + //case idealUint: + //case complex64 + //case complex128 + //case float32: + //case float64: + //case int8: + //case int16: + //case int32: + //case int64: + //case string: + //case uint8: + //case uint16: + //case uint32: + //case uint64: + //case *big.Int: + //case *big.Rat: + //case time.Duration: + case time.Time: + return x, nil + default: + return invConv(val, typ) + } + default: + log.Panic("internal error 006") + } + //dbg("%T(%v) %s", val, val, typeStr(typ)) + panic("unreachable") +} + +func invShiftRHS(lhs, rhs interface{}) (interface{}, error) { + return nil, fmt.Errorf("invalid operation: %v << %v (shift count type %T, must be unsigned integer)", lhs, rhs, rhs) +} + +func invTruncInt(v interface{}) error { + return fmt.Errorf("constant %v truncated to integer", v) +} + +func overflow(v interface{}, typ int) error { + return fmt.Errorf("constant %v overflows %s", v, typeStr(typ)) +} + +func typeCheck(rec []interface{}, cols []*col) (err error) { + for _, c := range cols { + i := c.index + if v := rec[i]; !c.typeCheck(v) { + switch v.(type) { + case idealComplex: + y := complex128(v.(idealComplex)) + switch c.typ { + case qBool: + case qComplex64: + rec[i] = complex64(y) + continue + case qComplex128: + rec[i] = complex128(y) + continue + case qFloat32, qFloat64, qInt8, qInt16, qInt32, qInt64, qUint8, qUint16, qUint32, qUint64: + return fmt.Errorf("constant %v truncated to real", y) + case qString: + default: + log.Panic("internal error 007") + } + case idealFloat: + y := float64(v.(idealFloat)) + switch c.typ { + case qBool: + case qComplex64: + rec[i] = complex(float32(y), 0) + continue + case qComplex128: + rec[i] = complex(float64(y), 0) + continue + case qFloat32: + rec[i] = float32(y) + continue + case qFloat64: + rec[i] = float64(y) + continue + case qInt8: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < math.MinInt8 || y > math.MaxInt8 { + return overflow(y, c.typ) + } + + rec[i] = int8(y) + continue + case qInt16: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < math.MinInt16 || y > math.MaxInt16 { + return overflow(y, c.typ) + } + + rec[i] = int16(y) + continue + case qInt32: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < math.MinInt32 || y > math.MaxInt32 { + return overflow(y, c.typ) + } + + rec[i] = int32(y) + continue + case qInt64: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < math.MinInt64 || y > math.MaxInt64 { + return overflow(y, c.typ) + } + + rec[i] = int64(y) + continue + case qString: + case qUint8: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < 0 || y > math.MaxUint8 { + return overflow(y, c.typ) + } + + rec[i] = uint8(y) + continue + case qUint16: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < 0 || y > math.MaxUint16 { + return overflow(y, c.typ) + } + + rec[i] = uint16(y) + continue + case qUint32: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < 0 || y > math.MaxUint32 { + return overflow(y, c.typ) + } + + rec[i] = uint32(y) + continue + case qUint64: + if math.Floor(y) != y { + return invTruncInt(y) + } + + if y < 0 || y > math.MaxUint64 { + return overflow(y, c.typ) + } + + rec[i] = uint64(y) + continue + case qBigInt: + if math.Floor(y) != y { + return invTruncInt(y) + } + + rr := big.NewRat(1, 1).SetFloat64(y) + ii := big.NewInt(0) + ii.Set(rr.Num()) + ii.Quo(ii, rr.Denom()) + rec[i] = ii + continue + case qBigRat: + rec[i] = big.NewRat(1, 1).SetFloat64(y) + continue + default: + log.Panic("internal error 008") + } + case idealInt: + y := int64(v.(idealInt)) + switch c.typ { + case qBool: + case qComplex64: + rec[i] = complex(float32(y), 0) + continue + case qComplex128: + rec[i] = complex(float64(y), 0) + continue + case qFloat32: + rec[i] = float32(y) + continue + case qFloat64: + rec[i] = float64(y) + continue + case qInt8: + if y < math.MinInt8 || y > math.MaxInt8 { + return overflow(y, c.typ) + } + + rec[i] = int8(y) + continue + case qInt16: + if y < math.MinInt16 || y > math.MaxInt16 { + return overflow(y, c.typ) + } + + rec[i] = int16(y) + continue + case qInt32: + if y < math.MinInt32 || y > math.MaxInt32 { + return overflow(y, c.typ) + } + + rec[i] = int32(y) + continue + case qInt64: + if y < math.MinInt64 || y > math.MaxInt64 { + return overflow(y, c.typ) + } + + rec[i] = int64(y) + continue + case qString: + case qUint8: + if y < 0 || y > math.MaxUint8 { + return overflow(y, c.typ) + } + + rec[i] = uint8(y) + continue + case qUint16: + if y < 0 || y > math.MaxUint16 { + return overflow(y, c.typ) + } + + rec[i] = uint16(y) + continue + case qUint32: + if y < 0 || y > math.MaxUint32 { + return overflow(y, c.typ) + } + + rec[i] = uint32(y) + continue + case qUint64: + if y < 0 { + return overflow(y, c.typ) + } + + rec[i] = uint64(y) + continue + case qBigInt: + rec[i] = big.NewInt(y) + continue + case qBigRat: + rec[i] = big.NewRat(1, 1).SetInt64(y) + continue + default: + log.Panic("internal error 009") + } + case idealRune: + y := int64(v.(idealRune)) + switch c.typ { + case qBool: + case qComplex64: + rec[i] = complex(float32(y), 0) + continue + case qComplex128: + rec[i] = complex(float64(y), 0) + continue + case qFloat32: + rec[i] = float32(y) + continue + case qFloat64: + rec[i] = float64(y) + continue + case qInt8: + if y < math.MinInt8 || y > math.MaxInt8 { + return overflow(y, c.typ) + } + + rec[i] = int8(y) + continue + case qInt16: + if y < math.MinInt16 || y > math.MaxInt16 { + return overflow(y, c.typ) + } + + rec[i] = int16(y) + continue + case qInt32: + if y < math.MinInt32 || y > math.MaxInt32 { + return overflow(y, c.typ) + } + + rec[i] = int32(y) + continue + case qInt64: + if y < math.MinInt64 || y > math.MaxInt64 { + return overflow(y, c.typ) + } + + rec[i] = int64(y) + continue + case qString: + case qUint8: + if y < 0 || y > math.MaxUint8 { + return overflow(y, c.typ) + } + + rec[i] = uint8(y) + continue + case qUint16: + if y < 0 || y > math.MaxUint16 { + return overflow(y, c.typ) + } + + rec[i] = uint16(y) + continue + case qUint32: + if y < 0 { + return overflow(y, c.typ) + } + + rec[i] = uint32(y) + continue + case qUint64: + if y < 0 { + return overflow(y, c.typ) + } + + rec[i] = uint64(y) + continue + case qBigInt: + rec[i] = big.NewInt(y) + continue + case qBigRat: + rec[i] = big.NewRat(1, 1).SetInt64(y) + continue + default: + log.Panic("internal error 010") + } + case idealUint: + y := uint64(v.(idealUint)) + switch c.typ { + case qBool: + case qComplex64: + rec[i] = complex(float32(y), 0) + continue + case qComplex128: + rec[i] = complex(float64(y), 0) + continue + case qFloat32: + rec[i] = float32(y) + continue + case qFloat64: + rec[i] = float64(y) + continue + case qInt8: + if y > math.MaxInt8 { + return overflow(y, c.typ) + } + + rec[i] = int8(y) + continue + case qInt16: + if y > math.MaxInt16 { + return overflow(y, c.typ) + } + + rec[i] = int16(y) + continue + case qInt32: + if y > math.MaxInt32 { + return overflow(y, c.typ) + } + + rec[i] = int32(y) + continue + case qInt64: + if y > math.MaxInt64 { + return overflow(y, c.typ) + } + + rec[i] = int64(y) + continue + case qString: + rec[i] = string(y) + continue + case qUint8: + if y > math.MaxUint8 { + return overflow(y, c.typ) + } + + rec[i] = uint8(y) + continue + case qUint16: + if y > math.MaxUint16 { + return overflow(y, c.typ) + } + + rec[i] = uint16(y) + continue + case qUint32: + if y > math.MaxUint32 { + return overflow(y, c.typ) + } + + rec[i] = uint32(y) + continue + case qUint64: + rec[i] = uint64(y) + continue + case qBigInt: + rec[i] = big.NewInt(0).SetUint64(y) + continue + case qBigRat: + ii := big.NewInt(0).SetUint64(y) + rec[i] = big.NewRat(1, 1).SetInt(ii) + continue + default: + log.Panic("internal error 011") + } + } + //dbg("v %T(%v), typ %s", v, v, typeStr(c.typ)) + return fmt.Errorf("cannot use %v (type %T) as %s in assignment/comparison to/with column %s", v, ideal(v), typeStr(c.typ), c.name) + } + } + return +} + +//TODO collate1 should return errors instead of panicing +func collate1(a, b interface{}) int { + switch x := a.(type) { + case nil: + if b != nil { + return -1 + } + + return 0 + case bool: + switch y := b.(type) { + case nil: + return 1 + case bool: + if !x && y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + default: + // Make bool collate before anything except nil and + // other bool for index seeking first non NULL value. + return -1 + } + case idealComplex: + switch y := b.(type) { + case nil: + return 1 + case idealComplex: + if x == y { + return 0 + } + + if real(x) < real(y) { + return -1 + } + + if real(x) > real(y) { + return 1 + } + + if imag(x) < imag(y) { + return -1 + } + + return 1 + case complex64: + { + x, y := complex64(x), complex64(y) + if x == y { + return 0 + } + + if real(x) < real(y) { + return -1 + } + + if real(x) > real(y) { + return 1 + } + + if imag(x) < imag(y) { + return -1 + } + + return 1 + } + case complex128: + { + x := complex128(x) + if x == y { + return 0 + } + + if real(x) < real(y) { + return -1 + } + + if real(x) > real(y) { + return 1 + } + + if imag(x) < imag(y) { + return -1 + } + + return 1 + } + default: + panic("internal error 012") + } + case idealUint: + switch y := b.(type) { + case nil: + return 1 + case idealUint: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case uint8: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case uint16: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case uint32: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case uint64: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case uint: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 013") + } + case idealRune: + switch y := b.(type) { + case nil: + return 1 + case idealRune: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case int8: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int16: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int32: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int64: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 014") + } + case idealInt: + switch y := b.(type) { + case nil: + return 1 + case idealInt: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case int8: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int16: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int32: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int64: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case int: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 015") + } + case idealFloat: + switch y := b.(type) { + case nil: + return 1 + case idealFloat: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case float32: + { + x, y := float64(x), float64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case float64: + { + x, y := float64(x), float64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 016") + } + case complex64: + switch y := b.(type) { + case nil: + return 1 + case complex64: + if x == y { + return 0 + } + + if real(x) < real(y) { + return -1 + } + + if real(x) > real(y) { + return 1 + } + + if imag(x) < imag(y) { + return -1 + } + + return 1 + case idealComplex: + { + x, y := complex64(x), complex64(y) + if x == y { + return 0 + } + + if real(x) < real(y) { + return -1 + } + + if real(x) > real(y) { + return 1 + } + + if imag(x) < imag(y) { + return -1 + } + + return 1 + } + default: + panic("internal error 017") + } + case complex128: + switch y := b.(type) { + case nil: + return 1 + case complex128: + if x == y { + return 0 + } + + if real(x) < real(y) { + return -1 + } + + if real(x) > real(y) { + return 1 + } + + if imag(x) < imag(y) { + return -1 + } + + return 1 + case idealComplex: + { + x, y := complex128(x), complex128(y) + if x == y { + return 0 + } + + if real(x) < real(y) { + return -1 + } + + if real(x) > real(y) { + return 1 + } + + if imag(x) < imag(y) { + return -1 + } + + return 1 + } + default: + panic("internal error 018") + } + case float32: + switch y := b.(type) { + case nil: + return 1 + case float32: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealFloat: + { + x, y := float32(x), float32(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 019") + } + case float64: + switch y := b.(type) { + case nil: + return 1 + case float64: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealFloat: + { + x, y := float64(x), float64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 020") + } + case int8: + switch y := b.(type) { + case nil: + return 1 + case int8: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 021") + } + case int16: + switch y := b.(type) { + case nil: + return 1 + case int16: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 022") + } + case int32: + switch y := b.(type) { + case nil: + return 1 + case int32: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 023") + } + case int64: + switch y := b.(type) { + case nil: + return 1 + case int64: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + //dbg("%T(%#v)", b, b) + panic("internal error 024") + } + case uint8: + switch y := b.(type) { + case nil: + return 1 + case uint8: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case idealUint: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 025") + } + case uint16: + switch y := b.(type) { + case nil: + return 1 + case uint16: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case idealUint: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 026") + } + case uint32: + switch y := b.(type) { + case nil: + return 1 + case uint32: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case idealUint: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 027") + } + case uint64: + switch y := b.(type) { + case nil: + return 1 + case uint64: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + case idealUint: + { + x, y := uint64(x), uint64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 028") + } + case string: + switch y := b.(type) { + case nil: + return 1 + case string: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + default: + panic("internal error 029") + } + case []byte: + switch y := b.(type) { + case nil: + return 1 + case []byte: + return bytes.Compare(x, y) + default: + panic("internal error 030") + } + case *big.Int: + switch y := b.(type) { + case nil: + return 1 + case *big.Int: + return x.Cmp(y) + case idealInt: + { + y := big.NewInt(int64(y)) + return x.Cmp(y) + } + case idealUint: + { + u := big.NewInt(0) + u.SetUint64(uint64(y)) + return x.Cmp(u) + } + default: + panic("internal error 031") + } + case *big.Rat: + switch y := b.(type) { + case nil: + return 1 + case *big.Rat: + return x.Cmp(y) + case idealInt: + { + y := big.NewRat(int64(y), 1) + return x.Cmp(y) + } + case idealUint: + { + u := big.NewInt(0) + u.SetUint64(uint64(y)) + var y big.Rat + y.SetInt(u) + return x.Cmp(&y) + } + default: + panic("internal error 032") + } + case time.Time: + switch y := b.(type) { + case nil: + return 1 + case time.Time: + if x.Before(y) { + return -1 + } + + if x.Equal(y) { + return 0 + } + + return 1 + default: + panic("internal error 033") + } + case time.Duration: + switch y := b.(type) { + case nil: + return 1 + case time.Duration: + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + default: + panic("internal error 034") + } + case chunk: + switch y := b.(type) { + case nil: + return 1 + case chunk: + a, err := x.expand() + if err != nil { + log.Panic(err) + } + + b, err := y.expand() + if err != nil { + log.Panic(err) + } + + return collate1(a, b) + default: + panic("internal error 035") + } + default: + //dbg("%T(%v) %T(%v)", a, a, b, b) + panic("internal error 036") + } +} + +//TODO collate should return errors from collate1 +func collate(x, y []interface{}) (r int) { + //defer func() { dbg("%v %v -> %v", x, y, r) }() + nx, ny := len(x), len(y) + + switch { + case nx == 0 && ny != 0: + return -1 + case nx == 0 && ny == 0: + return 0 + case nx != 0 && ny == 0: + return 1 + } + + r = 1 + if nx > ny { + x, y, r = y, x, -r + } + + for i, xi := range x { + if c := collate1(xi, y[i]); c != 0 { + return c * r + } + } + + if nx == ny { + return 0 + } + + return -r +} + +var collators = map[bool]func(a, b []interface{}) int{false: collateDesc, true: collate} + +func collateDesc(a, b []interface{}) int { + return -collate(a, b) +} + +func isOrderedType(v interface{}) (y interface{}, r bool, err error) { + //dbg("====") + //dbg("%T(%v)", v, v) + //defer func() { dbg("%T(%v)", y, y) }() + switch x := v.(type) { + case idealFloat, idealInt, idealRune, idealUint, + float32, float64, + int8, int16, int32, int64, + uint8, uint16, uint32, uint64, + string: + return v, true, nil + case *big.Int, *big.Rat, time.Time, time.Duration: + return x, true, nil + case chunk: + if y, err = x.expand(); err != nil { + return + } + + return isOrderedType(y) + } + + return v, false, nil +} + +var isSystemName = map[string]bool{ + "__Column": true, + "__Index": true, + "__Table": true, +} + +func qualifier(s string) string { + if pos := strings.IndexByte(s, '.'); pos >= 0 { + s = s[:pos] + } + return s +} + +func mustQualifier(s string) string { + q := qualifier(s) + if q == s { + panic("internal error 068") + } + + return q +} + +func selector(s string) string { + if pos := strings.IndexByte(s, '.'); pos >= 0 { + s = s[pos+1:] + } + return s +} + +func mustSelector(s string) string { + q := selector(s) + if q == s { + panic("internal error 069") + } + + return q +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/expr.go b/Godeps/_workspace/src/github.com/cznic/ql/expr.go new file mode 100644 index 00000000000..221a85ba9a6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/expr.go @@ -0,0 +1,3653 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found pIn the LICENSE file. + +package ql + +import ( + "fmt" + "log" + "math/big" + "regexp" + "strings" + "time" +) + +var ( + _ expression = (*binaryOperation)(nil) + _ expression = (*call)(nil) + _ expression = (*conversion)(nil) + _ expression = (*ident)(nil) + _ expression = (*indexOp)(nil) + _ expression = (*isNull)(nil) + _ expression = (*pIn)(nil) + _ expression = (*pLike)(nil) + _ expression = (*parameter)(nil) + _ expression = (*pexpr)(nil) + _ expression = (*slice)(nil) + _ expression = (*unaryOperation)(nil) + _ expression = value{} +) + +type expression interface { + eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) + isStatic() bool + String() string +} + +func staticExpr(e expression) (expression, error) { + if e.isStatic() { + v, err := e.eval(nil, nil, nil) + if err != nil { + return nil, err + } + + if v == nil { + return value{nil}, nil + } + + return value{v}, nil + } + + return e, nil +} + +type ( + idealComplex complex128 + idealFloat float64 + idealInt int64 + idealRune int32 + idealUint uint64 +) + +type exprTab struct { + expr expression + table string +} + +func isPossiblyRewriteableCrossJoinWhereExpression(expr expression) (bool, []exprTab) { + //dbg("....\n\texpr %v", expr) + //defer func() { dbg("\t\t%v: %v %v", expr, TODOb, TODOl) }() + switch x := expr.(type) { + case *binaryOperation: + if ok, tab, nx := x.isQIdentRelOpFixedValue(); ok { + return true, []exprTab{{nx, tab}} + } + + if x.op != andand { + return false, nil + } + + ok, rlist := isPossiblyRewriteableCrossJoinWhereExpression(x.r) + if !ok { + return false, nil + } + + ok, llist := isPossiblyRewriteableCrossJoinWhereExpression(x.l) + if !ok { + return false, nil + } + + return true, append(llist, rlist...) + case *ident: + if !x.isQualified() { + return false, nil + } + + return true, []exprTab{{&ident{mustSelector(x.s)}, mustQualifier(x.s)}} + case *unaryOperation: + ok, tab, nx := x.isNotQIdent() + if !ok { + return false, nil + } + + return true, []exprTab{{nx, tab}} + default: + //dbg("%T: %v", x, x) + return false, nil + } +} + +type pexpr struct { + expr expression +} + +func (p *pexpr) isStatic() bool { return p.expr.isStatic() } + +func (p *pexpr) String() string { + return fmt.Sprintf("(%s)", p.expr) +} + +func (p *pexpr) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + return p.expr.eval(execCtx, ctx, arg) +} + +//DONE newBetween +//LATER like newBetween, check all others have and use new* + +func newBetween(expr, lo, hi interface{}, not bool) (expression, error) { + e, err := staticExpr(expr.(expression)) + if err != nil { + return nil, err + } + + l, err := staticExpr(lo.(expression)) + if err != nil { + return nil, err + } + + h, err := staticExpr(hi.(expression)) + if err != nil { + return nil, err + } + + var a, b expression + op := andand + switch { + case not: // e < l || e > h + op = oror + if a, err = newBinaryOperation('<', e, l); err != nil { + return nil, err + } + + if b, err = newBinaryOperation('>', e, h); err != nil { + return nil, err + } + default: // e >= l && e <= h + if a, err = newBinaryOperation(ge, e, l); err != nil { + return nil, err + } + + if b, err = newBinaryOperation(le, e, h); err != nil { + return nil, err + } + } + + if a, err = staticExpr(a); err != nil { + return nil, err + } + + if b, err = staticExpr(b); err != nil { + return nil, err + } + + ret, err := newBinaryOperation(op, a, b) + if err != nil { + return nil, err + } + + return staticExpr(ret) +} + +type pLike struct { + expr expression + pattern expression + re *regexp.Regexp + sexpr *string +} + +func (p *pLike) isStatic() bool { return p.expr.isStatic() && p.pattern.isStatic() } +func (p *pLike) String() string { return fmt.Sprintf("%q LIKE %q", p.expr, p.pattern) } + +func (p *pLike) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + var sexpr string + var ok bool + switch { + case p.sexpr != nil: + sexpr = *p.sexpr + default: + expr, err := expand1(p.expr.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + if expr == nil { + return nil, nil + } + + sexpr, ok = expr.(string) + if !ok { + return nil, fmt.Errorf("non-string expression in LIKE: %v (value of type %T)", expr, expr) + } + + if p.expr.isStatic() { + p.sexpr = new(string) + *p.sexpr = sexpr + } + } + + re := p.re + if re == nil { + pattern, err := expand1(p.pattern.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + if pattern == nil { + return nil, nil + } + + spattern, ok := pattern.(string) + if !ok { + return nil, fmt.Errorf("non-string pattern in LIKE: %v (value of type %T)", pattern, pattern) + } + + if re, err = regexp.Compile(spattern); err != nil { + return nil, err + } + + if p.pattern.isStatic() { + p.re = re + } + } + + return re.MatchString(sexpr), nil +} + +type binaryOperation struct { + op int + l, r expression +} + +func newBinaryOperation(op int, x, y interface{}) (v expression, err error) { + b := binaryOperation{op, x.(expression), y.(expression)} + //dbg("newBinaryOperation %s", &b) + //defer func() { dbg("newBinaryOperation -> %v, %v", v, err) }() + var lv interface{} + if e := b.l; e.isStatic() { + if lv, err = e.eval(nil, nil, nil); err != nil { + return nil, err + } + + b.l = value{lv} + } + + if e := b.r; e.isStatic() { + v, err := e.eval(nil, nil, nil) + if err != nil { + return nil, err + } + + if v == nil { + return value{nil}, nil + } + + if op == '/' || op == '%' { + rb := binaryOperation{eq, e, value{idealInt(0)}} + val, err := rb.eval(nil, nil, nil) + if err != nil { + return nil, err + } + + if val.(bool) { + return nil, errDivByZero + } + } + + if b.l.isStatic() && lv == nil { + return value{nil}, nil + } + + b.r = value{v} + } + + if !b.isStatic() { + return &b, nil + } + + val, err := b.eval(nil, nil, nil) + return value{val}, err +} + +func (o *binaryOperation) isRelOp() bool { + op := o.op + return op == '<' || op == le || op == eq || op == neq || op == ge || op == '>' +} + +// [!]qident relOp fixedValue or vice versa +func (o *binaryOperation) isQIdentRelOpFixedValue() ( /* ok */ bool /* tableName */, string, expression) { + if !o.isRelOp() { + return false, "", nil + } + + switch lhs := o.l.(type) { + case *unaryOperation: + ok, tab, nx := lhs.isNotQIdent() + if !ok { + return false, "", nil + } + + switch rhs := o.r.(type) { + case *parameter, value: + return true, tab, &binaryOperation{o.op, nx, rhs} + } + case *ident: + if !lhs.isQualified() { + return false, "", nil + } + + switch rhs := o.r.(type) { + case *parameter, value: + return true, mustQualifier(lhs.s), &binaryOperation{o.op, &ident{mustSelector(lhs.s)}, rhs} + } + case *parameter, value: + switch rhs := o.r.(type) { + case *ident: + if !rhs.isQualified() { + return false, "", nil + } + + return true, mustQualifier(rhs.s), &binaryOperation{o.op, lhs, &ident{mustSelector(rhs.s)}} + case *unaryOperation: + ok, tab, nx := rhs.isNotQIdent() + if !ok { + return false, "", nil + } + + return true, tab, &binaryOperation{o.op, lhs, nx} + } + } + return false, "", nil +} + +func (o *binaryOperation) isBoolAnd() bool { return o.op == andand } + +func (o *binaryOperation) isStatic() bool { return o.l.isStatic() && o.r.isStatic() } + +func (o *binaryOperation) String() string { + return fmt.Sprintf("%s%s%s", o.l, iop(o.op), o.r) +} + +func (o *binaryOperation) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (r interface{}, err error) { + defer func() { + if e := recover(); e != nil { + switch x := e.(type) { + case error: + r, err = nil, x + default: + r, err = nil, fmt.Errorf("%v", x) + } + } + }() + + switch op := o.op; op { + case andand: + a, err := expand1(o.l.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + switch x := a.(type) { + case nil: + b, err := expand1(o.r.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + switch y := b.(type) { + case nil: + return nil, nil + case bool: + if !y { + return false, nil + } + + return nil, nil + default: + return invOp2(x, y, op) + } + case bool: + if !x { + return false, nil + } + + b, err := expand1(o.r.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + switch y := b.(type) { + case nil: + return nil, nil + case bool: + return y, nil + default: + return invOp2(x, y, op) + } + default: + return undOp(x, op) + } + case oror: + a, err := expand1(o.l.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + switch x := a.(type) { + case nil: + b, err := expand1(o.r.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + switch y := b.(type) { + case nil: + return nil, nil + case bool: + if y { + return y, nil + } + + return nil, nil + default: + return invOp2(x, y, op) + } + case bool: + if x { + return x, nil + } + + b, err := expand1(o.r.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + switch y := b.(type) { + case nil: + return nil, nil + case bool: + return y, nil + default: + return invOp2(x, y, op) + } + default: + return undOp(x, op) + } + case '>': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + switch y := b.(type) { + case idealFloat: + return x > y, nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return x > y, nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return x > y, nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return x > y, nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + switch y := b.(type) { + case float32: + return x > y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x > y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x > y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x > y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x > y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x > y, nil + default: + return invOp2(x, y, op) + } + case string: + switch y := b.(type) { + case string: + return x > y, nil + default: + return invOp2(x, y, op) + } + case uint8: + switch y := b.(type) { + case uint8: + return x > y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x > y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x > y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x > y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + return x.Cmp(y) > 0, nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + return x.Cmp(y) > 0, nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x > y, nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Time: + return x.After(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '<': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + switch y := b.(type) { + case idealFloat: + return x < y, nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return x < y, nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return x < y, nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return x < y, nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + switch y := b.(type) { + case float32: + return x < y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x < y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x < y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x < y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x < y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x < y, nil + default: + return invOp2(x, y, op) + } + case string: + switch y := b.(type) { + case string: + return x < y, nil + default: + return invOp2(x, y, op) + } + case uint8: + switch y := b.(type) { + case uint8: + return x < y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x < y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x < y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x < y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + return x.Cmp(y) < 0, nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + return x.Cmp(y) < 0, nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x < y, nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Time: + return x.Before(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case le: + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + switch y := b.(type) { + case idealFloat: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + switch y := b.(type) { + case float32: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case string: + switch y := b.(type) { + case string: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case uint8: + switch y := b.(type) { + case uint8: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + return x.Cmp(y) <= 0, nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + return x.Cmp(y) <= 0, nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x <= y, nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Time: + return x.Before(y) || x.Equal(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case ge: + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + switch y := b.(type) { + case idealFloat: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + switch y := b.(type) { + case float32: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case string: + switch y := b.(type) { + case string: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case uint8: + switch y := b.(type) { + case uint8: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + return x.Cmp(y) >= 0, nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + return x.Cmp(y) >= 0, nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x >= y, nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Time: + return x.After(y) || x.Equal(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case neq: + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + switch y := b.(type) { + case idealComplex: + return x != y, nil + default: + return invOp2(x, y, op) + } + case idealFloat: + switch y := b.(type) { + case idealFloat: + return x != y, nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return x != y, nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return x != y, nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return x != y, nil + default: + return invOp2(x, y, op) + } + case bool: + switch y := b.(type) { + case bool: + return x != y, nil + default: + return invOp2(x, y, op) + } + case complex64: + switch y := b.(type) { + case complex64: + return x != y, nil + default: + return invOp2(x, y, op) + } + case complex128: + switch y := b.(type) { + case complex128: + return x != y, nil + default: + return invOp2(x, y, op) + } + case float32: + switch y := b.(type) { + case float32: + return x != y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x != y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x != y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x != y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x != y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x != y, nil + default: + return invOp2(x, y, op) + } + case string: + switch y := b.(type) { + case string: + return x != y, nil + default: + return invOp2(x, y, op) + } + case uint8: + switch y := b.(type) { + case uint8: + return x != y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x != y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x != y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x != y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + return x.Cmp(y) != 0, nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + return x.Cmp(y) != 0, nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x != y, nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Time: + return !x.Equal(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case eq: + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + switch y := b.(type) { + case idealComplex: + return x == y, nil + default: + return invOp2(x, y, op) + } + case idealFloat: + switch y := b.(type) { + case idealFloat: + return x == y, nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return x == y, nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return x == y, nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return x == y, nil + default: + return invOp2(x, y, op) + } + case bool: + switch y := b.(type) { + case bool: + return x == y, nil + default: + return invOp2(x, y, op) + } + case complex64: + switch y := b.(type) { + case complex64: + return x == y, nil + default: + return invOp2(x, y, op) + } + case complex128: + switch y := b.(type) { + case complex128: + return x == y, nil + default: + return invOp2(x, y, op) + } + case float32: + switch y := b.(type) { + case float32: + return x == y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x == y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x == y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x == y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x == y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x == y, nil + default: + return invOp2(x, y, op) + } + case string: + switch y := b.(type) { + case string: + return x == y, nil + default: + return invOp2(x, y, op) + } + case uint8: + switch y := b.(type) { + case uint8: + return x == y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x == y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x == y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x == y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + return x.Cmp(y) == 0, nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + return x.Cmp(y) == 0, nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x == y, nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Time: + return x.Equal(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '+': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + switch y := b.(type) { + case idealComplex: + return idealComplex(complex64(x) + complex64(y)), nil + default: + return invOp2(x, y, op) + } + case idealFloat: + switch y := b.(type) { + case idealFloat: + return idealFloat(float64(x) + float64(y)), nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) + int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) + int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) + uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + switch y := b.(type) { + case complex64: + return x + y, nil + default: + return invOp2(x, y, op) + } + case complex128: + switch y := b.(type) { + case complex128: + return x + y, nil + default: + return invOp2(x, y, op) + } + case float32: + switch y := b.(type) { + case float32: + return x + y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x + y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x + y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x + y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x + y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x + y, nil + default: + return invOp2(x, y, op) + } + case string: + switch y := b.(type) { + case string: + return x + y, nil + default: + return invOp2(x, y, op) + } + case uint8: + switch y := b.(type) { + case uint8: + return x + y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x + y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x + y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x + y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + var z big.Int + return z.Add(x, y), nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + var z big.Rat + return z.Add(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x + y, nil + case time.Time: + return y.Add(x), nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Duration: + return x.Add(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '-': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + switch y := b.(type) { + case idealComplex: + return idealComplex(complex64(x) - complex64(y)), nil + default: + return invOp2(x, y, op) + } + case idealFloat: + switch y := b.(type) { + case idealFloat: + return idealFloat(float64(x) - float64(y)), nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) - int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) - int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) - uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + switch y := b.(type) { + case complex64: + return x - y, nil + default: + return invOp2(x, y, op) + } + case complex128: + switch y := b.(type) { + case complex128: + return x - y, nil + default: + return invOp2(x, y, op) + } + case float32: + switch y := b.(type) { + case float32: + return x - y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x - y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x - y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x - y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x - y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x - y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x - y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x - y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x - y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x - y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + var z big.Int + return z.Sub(x, y), nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + var z big.Rat + return z.Sub(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x - y, nil + default: + return invOp2(x, y, op) + } + case time.Time: + switch y := b.(type) { + case time.Duration: + return x.Add(-y), nil + case time.Time: + return x.Sub(y), nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case rsh: + a, b := eval2(o.l, o.r, execCtx, ctx, arg) + if a == nil || b == nil { + return + } + + var cnt uint64 + switch y := b.(type) { + //case nil: + case idealComplex: + return invShiftRHS(a, b) + case idealFloat: + return invShiftRHS(a, b) + case idealInt: + cnt = uint64(y) + case idealRune: + cnt = uint64(y) + case idealUint: + cnt = uint64(y) + case bool: + return invShiftRHS(a, b) + case complex64: + return invShiftRHS(a, b) + case complex128: + return invShiftRHS(a, b) + case float32: + return invShiftRHS(a, b) + case float64: + return invShiftRHS(a, b) + case int8: + return invShiftRHS(a, b) + case int16: + return invShiftRHS(a, b) + case int32: + return invShiftRHS(a, b) + case int64: + return invShiftRHS(a, b) + case string: + return invShiftRHS(a, b) + case uint8: + cnt = uint64(y) + case uint16: + cnt = uint64(y) + case uint32: + cnt = uint64(y) + case uint64: + cnt = uint64(y) + default: + return invOp2(a, b, op) + } + + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + return undOp2(a, b, op) + case idealInt: + return idealInt(int64(x) >> cnt), nil + case idealRune: + return idealRune(int64(x) >> cnt), nil + case idealUint: + return idealUint(uint64(x) >> cnt), nil + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + return undOp2(a, b, op) + case float64: + return undOp2(a, b, op) + case int8: + return x >> cnt, nil + case int16: + return x >> cnt, nil + case int32: + return x >> cnt, nil + case int64: + return x >> cnt, nil + case string: + return undOp2(a, b, op) + case uint8: + return x >> cnt, nil + case uint16: + return x >> cnt, nil + case uint32: + return x >> cnt, nil + case uint64: + return x >> cnt, nil + case *big.Int: + var z big.Int + return z.Rsh(x, uint(cnt)), nil + case time.Duration: + return x >> cnt, nil + default: + return invOp2(a, b, op) + } + case lsh: + a, b := eval2(o.l, o.r, execCtx, ctx, arg) + if a == nil || b == nil { + return + } + + var cnt uint64 + switch y := b.(type) { + //case nil: + case idealComplex: + return invShiftRHS(a, b) + case idealFloat: + return invShiftRHS(a, b) + case idealInt: + cnt = uint64(y) + case idealRune: + cnt = uint64(y) + case idealUint: + cnt = uint64(y) + case bool: + return invShiftRHS(a, b) + case complex64: + return invShiftRHS(a, b) + case complex128: + return invShiftRHS(a, b) + case float32: + return invShiftRHS(a, b) + case float64: + return invShiftRHS(a, b) + case int8: + return invShiftRHS(a, b) + case int16: + return invShiftRHS(a, b) + case int32: + return invShiftRHS(a, b) + case int64: + return invShiftRHS(a, b) + case string: + return invShiftRHS(a, b) + case uint8: + cnt = uint64(y) + case uint16: + cnt = uint64(y) + case uint32: + cnt = uint64(y) + case uint64: + cnt = uint64(y) + default: + return invOp2(a, b, op) + } + + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + return undOp2(a, b, op) + case idealInt: + return idealInt(int64(x) << cnt), nil + case idealRune: + return idealRune(int64(x) << cnt), nil + case idealUint: + return idealUint(uint64(x) << cnt), nil + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + return undOp2(a, b, op) + case float64: + return undOp2(a, b, op) + case int8: + return x << cnt, nil + case int16: + return x << cnt, nil + case int32: + return x << cnt, nil + case int64: + return x << cnt, nil + case string: + return undOp2(a, b, op) + case uint8: + return x << cnt, nil + case uint16: + return x << cnt, nil + case uint32: + return x << cnt, nil + case uint64: + return x << cnt, nil + case *big.Int: + var z big.Int + return z.Lsh(x, uint(cnt)), nil + case time.Duration: + return x << cnt, nil + default: + return invOp2(a, b, op) + } + case '&': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + return undOp2(a, b, op) + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) & int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) & int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) & uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + return undOp2(a, b, op) + case float64: + return undOp2(a, b, op) + case int8: + switch y := b.(type) { + case int8: + return x & y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x & y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x & y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x & y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x & y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x & y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x & y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x & y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + var z big.Int + return z.And(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x & y, nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '|': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + return undOp2(a, b, op) + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) | int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) | int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) | uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + return undOp2(a, b, op) + case float64: + return undOp2(a, b, op) + case int8: + switch y := b.(type) { + case int8: + return x | y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x | y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x | y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x | y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x | y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x | y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x | y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x | y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + var z big.Int + return z.Or(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x | y, nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case andnot: + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + return undOp2(a, b, op) + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) &^ int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) &^ int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) &^ uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + return undOp2(a, b, op) + case float64: + return undOp2(a, b, op) + case int8: + switch y := b.(type) { + case int8: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + var z big.Int + return z.AndNot(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x &^ y, nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '^': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + return undOp2(a, b, op) + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) ^ int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) ^ int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) ^ uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + return undOp2(a, b, op) + case float64: + return undOp2(a, b, op) + case int8: + switch y := b.(type) { + case int8: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + var z big.Int + return z.Xor(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x ^ y, nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '%': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp2(a, b, op) + case idealFloat: + return undOp2(a, b, op) + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) % int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) % int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) % uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + return undOp2(a, b, op) + case complex128: + return undOp2(a, b, op) + case float32: + return undOp2(a, b, op) + case float64: + return undOp2(a, b, op) + case int8: + switch y := b.(type) { + case int8: + return x % y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x % y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x % y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x % y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x % y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x % y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x % y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x % y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + if y.Sign() == 0 { + return nil, errDivByZero + } + + var z big.Int + return z.Mod(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x % y, nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '/': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + switch y := b.(type) { + case idealComplex: + return idealComplex(complex64(x) / complex64(y)), nil + default: + return invOp2(x, y, op) + } + case idealFloat: + switch y := b.(type) { + case idealFloat: + return idealFloat(float64(x) / float64(y)), nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) / int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) / int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) / uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + switch y := b.(type) { + case complex64: + return x / y, nil + default: + return invOp2(x, y, op) + } + case complex128: + switch y := b.(type) { + case complex128: + return x / y, nil + default: + return invOp2(x, y, op) + } + case float32: + switch y := b.(type) { + case float32: + return x / y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x / y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x / y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x / y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x / y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x / y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x / y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x / y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x / y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x / y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + if y.Sign() == 0 { + return nil, errDivByZero + } + + var z big.Int + return z.Quo(x, y), nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + if y.Sign() == 0 { + return nil, errDivByZero + } + + var z big.Rat + return z.Quo(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x / y, nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + case '*': + a, b := o.get2(execCtx, ctx, arg) + if a == nil || b == nil { + return + } + switch x := a.(type) { + //case nil: + case idealComplex: + switch y := b.(type) { + case idealComplex: + return idealComplex(complex64(x) * complex64(y)), nil + default: + return invOp2(x, y, op) + } + case idealFloat: + switch y := b.(type) { + case idealFloat: + return idealFloat(float64(x) * float64(y)), nil + default: + return invOp2(x, y, op) + } + case idealInt: + switch y := b.(type) { + case idealInt: + return idealInt(int64(x) * int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealRune: + switch y := b.(type) { + case idealRune: + return idealRune(int64(x) * int64(y)), nil + default: + return invOp2(x, y, op) + } + case idealUint: + switch y := b.(type) { + case idealUint: + return idealUint(uint64(x) * uint64(y)), nil + default: + return invOp2(x, y, op) + } + case bool: + return undOp2(a, b, op) + case complex64: + switch y := b.(type) { + case complex64: + return x * y, nil + default: + return invOp2(x, y, op) + } + case complex128: + switch y := b.(type) { + case complex128: + return x * y, nil + default: + return invOp2(x, y, op) + } + case float32: + switch y := b.(type) { + case float32: + return x * y, nil + default: + return invOp2(x, y, op) + } + case float64: + switch y := b.(type) { + case float64: + return x * y, nil + default: + return invOp2(x, y, op) + } + case int8: + switch y := b.(type) { + case int8: + return x * y, nil + default: + return invOp2(x, y, op) + } + case int16: + switch y := b.(type) { + case int16: + return x * y, nil + default: + return invOp2(x, y, op) + } + case int32: + switch y := b.(type) { + case int32: + return x * y, nil + default: + return invOp2(x, y, op) + } + case int64: + switch y := b.(type) { + case int64: + return x * y, nil + default: + return invOp2(x, y, op) + } + case string: + return undOp2(a, b, op) + case uint8: + switch y := b.(type) { + case uint8: + return x * y, nil + default: + return invOp2(x, y, op) + } + case uint16: + switch y := b.(type) { + case uint16: + return x * y, nil + default: + return invOp2(x, y, op) + } + case uint32: + switch y := b.(type) { + case uint32: + return x * y, nil + default: + return invOp2(x, y, op) + } + case uint64: + switch y := b.(type) { + case uint64: + return x * y, nil + default: + return invOp2(x, y, op) + } + case *big.Int: + switch y := b.(type) { + case *big.Int: + var z big.Int + return z.Mul(x, y), nil + default: + return invOp2(x, y, op) + } + case *big.Rat: + switch y := b.(type) { + case *big.Rat: + var z big.Rat + return z.Mul(x, y), nil + default: + return invOp2(x, y, op) + } + case time.Duration: + switch y := b.(type) { + case time.Duration: + return x * y, nil + default: + return invOp2(x, y, op) + } + default: + return invOp2(a, b, op) + } + default: + log.Panic("internal error 037") + panic("unreachable") + } +} + +func (o *binaryOperation) get2(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (x, y interface{}) { + x, y = eval2(o.l, o.r, execCtx, ctx, arg) + //dbg("get2 pIn - ", x, y) + //defer func() {dbg("get2 coerced ", x, y)}() + return coerce(x, y) +} + +type ident struct { + s string +} + +func (i *ident) isQualified() bool { return strings.Contains(i.s, ".") } + +func (i *ident) isStatic() bool { return false } + +func (i *ident) String() string { return i.s } + +func (i *ident) eval(execCtx *execCtx, ctx map[interface{}]interface{}, _ []interface{}) (v interface{}, err error) { + if _, ok := ctx["$agg0"]; ok { + return int64(0), nil + } + + //defer func() { dbg("ident %q -> %v %v", i.s, v, err) }() + v, ok := ctx[i.s] + if !ok { + err = fmt.Errorf("unknown field %s", i.s) + } + return +} + +type pIn struct { + expr expression + list []expression + m map[interface{}]struct{} // IN (SELECT...) results + not bool + sample interface{} + sel *selectStmt +} + +func (n *pIn) isStatic() bool { + if !n.expr.isStatic() || n.sel != nil { + return false + } + + for _, v := range n.list { + if !v.isStatic() { + return false + } + } + return true +} + +//LATER newIn + +func (n *pIn) String() string { + if n.sel == nil { + a := []string{} + for _, v := range n.list { + a = append(a, v.String()) + } + if n.not { + return fmt.Sprintf("%s NOT IN (%s)", n.expr, strings.Join(a, ",")) + } + + return fmt.Sprintf("%s IN (%s)", n.expr, strings.Join(a, ",")) + } + + if n.not { + return fmt.Sprintf("%s NOT IN (%s)", n.expr, n.sel) + } + + return fmt.Sprintf("%s IN (%s)", n.expr, n.sel) +} + +func (n *pIn) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + lhs, err := expand1(n.expr.eval(execCtx, ctx, arg)) + if err != nil { + return nil, err + } + + if lhs == nil { + return nil, nil //TODO Add test for NULL LHS. + } + + if n.sel == nil { + for _, v := range n.list { + b, err := newBinaryOperation(eq, value{lhs}, v) + if err != nil { + return nil, err + } + + eval, err := b.eval(execCtx, ctx, arg) + if err != nil { + return nil, err + } + + if x, ok := eval.(bool); ok && x { + return !n.not, nil + } + } + return n.not, nil + } + + if n.m == nil { // SELECT not yet evaluated. + r := n.sel.exec0() + n.m = map[interface{}]struct{}{} + ok := false + typechecked := false + if err := r.do(execCtx, false, func(id interface{}, data []interface{}) (more bool, err error) { + if typechecked { + if data[0] == nil { + return true, nil + } + + n.m[data[0]] = struct{}{} + } + + if ok { + if data[0] == nil { + return true, nil + } + + n.sample = data[0] + switch n.sample.(type) { + case bool, byte, complex128, complex64, float32, + float64, int16, int32, int64, int8, + string, uint16, uint32, uint64: + typechecked = true + n.m[n.sample] = struct{}{} + return true, nil + default: + return false, fmt.Errorf("IN (%s): invalid field type: %T", n.sel, data[0]) + } + + } + + flds := data[0].([]*fld) + if g, e := len(flds), 1; g != e { + return false, fmt.Errorf("IN (%s): mismatched field count, have %d, need %d", n.sel, g, e) + } + + ok = true + return true, nil + }); err != nil { + return nil, err + } + } + + if n.sample == nil { + return nil, nil + } + + _, ok := n.m[coerce1(lhs, n.sample)] + return ok != n.not, nil +} + +type value struct { + val interface{} +} + +func (l value) isStatic() bool { return true } + +func (l value) String() string { + switch x := l.val.(type) { + case nil: + return "NULL" + case string: + return fmt.Sprintf("%q", x) + default: + return fmt.Sprintf("%v", l.val) + } +} + +func (l value) eval(execCtx *execCtx, ctx map[interface{}]interface{}, _ []interface{}) (interface{}, error) { + return l.val, nil +} + +type conversion struct { + typ int + val expression +} + +func (c *conversion) isStatic() bool { + return c.val.isStatic() +} + +//LATER newConversion or fake unary op + +func (c *conversion) String() string { + return fmt.Sprintf("%s(%s)", typeStr(c.typ), c.val) +} + +func (c *conversion) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + val, err := expand1(c.val.eval(execCtx, ctx, arg)) + if err != nil { + return + } + + return convert(val, c.typ) +} + +type unaryOperation struct { + op int + v expression +} + +func newUnaryOperation(op int, x interface{}) (v expression, err error) { + l, ok := x.(expression) + if !ok { + log.Panic("internal error 038") + } + + u := unaryOperation{op, l} + if !l.isStatic() { + return &u, nil + } + + val, err := u.eval(nil, nil, nil) + if val == nil { + return value{nil}, nil + } + + return value{val}, err +} + +func (u *unaryOperation) isStatic() bool { return u.v.isStatic() } + +func (u *unaryOperation) String() string { return fmt.Sprintf("%s%s", iop(u.op), u.v) } + +// !ident +func (u *unaryOperation) isNotQIdent() (bool, string, expression) { + if u.op != '!' { + return false, "", nil + } + + id, ok := u.v.(*ident) + if ok && id.isQualified() { + return true, mustQualifier(id.s), &unaryOperation{'!', &ident{mustSelector(id.s)}} + } + + return false, "", nil +} + +func (u *unaryOperation) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (r interface{}, err error) { + defer func() { + if e := recover(); e != nil { + switch x := e.(type) { + case error: + r, err = nil, x + default: + r, err = nil, fmt.Errorf("%v", x) + } + } + }() + + switch op := u.op; op { + case '!': + a := eval(u.v, execCtx, ctx, arg) + if a == nil { + return + } + + switch x := a.(type) { + case bool: + return !x, nil + default: + return undOp(a, op) + } + case '^': + a := eval(u.v, execCtx, ctx, arg) + if a == nil { + return + } + + switch x := a.(type) { + //case nil: + case idealComplex: + return undOp(a, op) + case idealFloat: + return undOp(a, op) + case idealInt: + return ^x, nil + case idealRune: + return ^x, nil + case idealUint: + return ^x, nil + case bool: + return undOp(a, op) + case complex64: + return undOp(a, op) + case complex128: + return undOp(a, op) + case float32: + return undOp(a, op) + case float64: + return undOp(a, op) + case int8: + return ^x, nil + case int16: + return ^x, nil + case int32: + return ^x, nil + case int64: + return ^x, nil + case string: + return undOp(a, op) + case uint8: + return ^x, nil + case uint16: + return ^x, nil + case uint32: + return ^x, nil + case uint64: + return ^x, nil + case *big.Int: + var z big.Int + return z.Not(x), nil + case time.Duration: + return ^x, nil + default: + return undOp(a, op) + } + case '+': + a := eval(u.v, execCtx, ctx, arg) + if a == nil { + return + } + + switch x := a.(type) { + //case nil: + case idealComplex: + return +x, nil + case idealFloat: + return +x, nil + case idealInt: + return +x, nil + case idealRune: + return +x, nil + case idealUint: + return +x, nil + case bool: + return undOp(a, op) + case complex64: + return +x, nil + case complex128: + return +x, nil + case float32: + return +x, nil + case float64: + return +x, nil + case int8: + return +x, nil + case int16: + return +x, nil + case int32: + return +x, nil + case int64: + return +x, nil + case string: + return undOp(a, op) + case uint8: + return +x, nil + case uint16: + return +x, nil + case uint32: + return +x, nil + case uint64: + return +x, nil + case *big.Int: + var z big.Int + return z.Set(x), nil + case *big.Rat: + var z big.Rat + return z.Set(x), nil + case time.Duration: + return x, nil + default: + return undOp(a, op) + } + case '-': + a := eval(u.v, execCtx, ctx, arg) + if a == nil { + return + } + + switch x := a.(type) { + //case nil: + case idealComplex: + return -x, nil + case idealFloat: + return -x, nil + case idealInt: + return -x, nil + case idealRune: + return -x, nil + case idealUint: + return -x, nil + case bool: + return undOp(a, op) + case complex64: + return -x, nil + case complex128: + return -x, nil + case float32: + return -x, nil + case float64: + return -x, nil + case int8: + return -x, nil + case int16: + return -x, nil + case int32: + return -x, nil + case int64: + return -x, nil + case string: + return undOp(a, op) + case uint8: + return -x, nil + case uint16: + return -x, nil + case uint32: + return -x, nil + case uint64: + return -x, nil + case *big.Int: + var z big.Int + return z.Neg(x), nil + case *big.Rat: + var z big.Rat + return z.Neg(x), nil + case time.Duration: + return -x, nil + default: + return undOp(a, op) + } + default: + log.Panic("internal error 039") + panic("unreachable") + } +} + +type call struct { + f string + arg []expression +} + +func newCall(f string, arg []expression) (v expression, isAgg bool, err error) { + x := builtin[f] + if x.f == nil { + return nil, false, fmt.Errorf("undefined: %s", f) + } + + isAgg = x.isAggregate + if g, min, max := len(arg), x.minArgs, x.maxArgs; g < min || g > max { + a := []interface{}{} + for _, v := range arg { + a = append(a, v) + } + return nil, false, badNArgs(min, f, a) + } + + c := call{f: f} + for _, val := range arg { + if !val.isStatic() { + c.arg = append(c.arg, val) + continue + } + + eval, err := val.eval(nil, nil, nil) + if err != nil { + return nil, isAgg, err + } + + c.arg = append(c.arg, value{eval}) + } + + return &c, isAgg, nil +} + +func (c *call) isStatic() bool { + v := builtin[c.f] + if v.f == nil || !v.isStatic { + return false + } + + for _, v := range c.arg { + if !v.isStatic() { + return false + } + } + return true +} + +func (c *call) String() string { + a := []string{} + for _, v := range c.arg { + a = append(a, v.String()) + } + return fmt.Sprintf("%s(%s)", c.f, strings.Join(a, ", ")) +} + +func (c *call) eval(execCtx *execCtx, ctx map[interface{}]interface{}, args []interface{}) (v interface{}, err error) { + f, ok := builtin[c.f] + if !ok { + return nil, fmt.Errorf("unknown function %s", c.f) + } + + isId := c.f == "id" + a := make([]interface{}, len(c.arg)) + for i, arg := range c.arg { + if v, err = expand1(arg.eval(execCtx, ctx, args)); err != nil { + if !isId { + return nil, err + } + + if _, ok := arg.(*ident); !ok { + return nil, err + } + + a[i] = arg + continue + } + + a[i] = v + } + + if ctx != nil { + ctx["$fn"] = c + } + return f.f(a, ctx) +} + +type parameter struct { + n int +} + +func (parameter) isStatic() bool { return false } + +func (p parameter) String() string { return fmt.Sprintf("$%d", p.n) } + +func (p parameter) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + i := p.n - 1 + if i < len(arg) { + return arg[i], nil + } + + return nil, fmt.Errorf("missing %s", p) +} + +//MAYBE make it an unary operation +type isNull struct { + expr expression + not bool +} + +//LATER newIsNull + +func (is *isNull) isStatic() bool { return is.expr.isStatic() } + +func (is *isNull) String() string { + if is.not { + return fmt.Sprintf("%s IS NOT NULL", is.expr) + } + + return fmt.Sprintf("%s IS NULL", is.expr) +} + +func (is *isNull) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + val, err := is.expr.eval(execCtx, ctx, arg) + if err != nil { + return + } + + return val == nil != is.not, nil +} + +type indexOp struct { + expr, x expression +} + +func newIndex(sv, xv expression) (v expression, err error) { + s, fs, i := "", false, uint64(0) + x := indexOp{sv, xv} + if x.expr.isStatic() { + v, err := x.expr.eval(nil, nil, nil) + if err != nil { + return nil, err + } + + if v == nil { + return value{nil}, nil + } + + if s, fs = v.(string); !fs { + return nil, invXOp(sv, xv) + } + + x.expr = value{s} + } + + if x.x.isStatic() { + v, err := x.x.eval(nil, nil, nil) + if err != nil { + return nil, err + } + + if v == nil { + return value{nil}, nil + } + + var p *string + if fs { + p = &s + } + if i, err = indexExpr(p, v); err != nil { + return nil, err + } + + x.x = value{i} + } + + return &x, nil +} + +func (x *indexOp) isStatic() bool { + return x.expr.isStatic() && x.x.isStatic() +} + +func (x *indexOp) String() string { return fmt.Sprintf("%s[%s]", x.expr, x.x) } + +func (x *indexOp) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + s0, err := x.expr.eval(execCtx, ctx, arg) + if err != nil { + return nil, runErr(err) + } + + s, ok := s0.(string) + if !ok { + return nil, runErr(invXOp(s0, x.x)) + } + + i0, err := x.x.eval(execCtx, ctx, arg) + if err != nil { + return nil, runErr(err) + } + + if i0 == nil { + return nil, nil + } + + i, err := indexExpr(&s, i0) + if err != nil { + return nil, runErr(err) + } + + return s[i], nil +} + +type slice struct { + expr expression + lo, hi *expression +} + +func newSlice(expr expression, lo, hi *expression) (v expression, err error) { + y := slice{expr, lo, hi} + var val interface{} + if e := y.expr; e.isStatic() { + if val, err = e.eval(nil, nil, nil); err != nil { + return nil, err + } + + if val == nil { + return value{nil}, nil + } + + y.expr = value{val} + } + + if p := y.lo; p != nil { + if e := *p; e.isStatic() { + if val, err = e.eval(nil, nil, nil); err != nil { + return nil, err + } + + if val == nil { + return value{nil}, nil + } + + v := expression(value{val}) + y.lo = &v + } + } + + if p := y.hi; p != nil { + if e := *p; e.isStatic() { + if val, err = e.eval(nil, nil, nil); err != nil { + return nil, err + } + + if val == nil { + return value{nil}, nil + } + + v := expression(value{val}) + y.hi = &v + } + } + return &y, nil +} + +func (s *slice) eval(execCtx *execCtx, ctx map[interface{}]interface{}, arg []interface{}) (v interface{}, err error) { + s0, err := s.expr.eval(execCtx, ctx, arg) + if err != nil { + return + } + + if s0 == nil { + return + } + + ss, ok := s0.(string) + if !ok { + return nil, runErr(invSOp(s0)) + } + + var iLo, iHi uint64 + if s.lo != nil { + i, err := (*s.lo).eval(execCtx, ctx, arg) + if err != nil { + return nil, err + } + + if i == nil { + return nil, err + } + + if iLo, err = sliceExpr(&ss, i, 0); err != nil { + return nil, err + } + } + + iHi = uint64(len(ss)) + if s.hi != nil { + i, err := (*s.hi).eval(execCtx, ctx, arg) + if err != nil { + return nil, err + } + + if i == nil { + return nil, err + } + + if iHi, err = sliceExpr(&ss, i, 1); err != nil { + return nil, err + } + } + + return ss[iLo:iHi], nil +} + +func (s *slice) isStatic() bool { + if !s.expr.isStatic() { + return false + } + + if p := s.lo; p != nil && !(*p).isStatic() { + return false + } + + if p := s.hi; p != nil && !(*p).isStatic() { + return false + } + + return false +} + +func (s *slice) String() string { + switch { + case s.lo == nil && s.hi == nil: + return fmt.Sprintf("%v[:]", s.expr) + case s.lo == nil && s.hi != nil: + return fmt.Sprintf("%v[:%v]", s.expr, *s.hi) + case s.lo != nil && s.hi == nil: + return fmt.Sprintf("%v[%v:]", s.expr, *s.lo) + default: //case s.lo != nil && s.hi != nil: + return fmt.Sprintf("%v[%v:%v]", s.expr, *s.lo, *s.hi) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/file.go b/Godeps/_workspace/src/github.com/cznic/ql/file.go new file mode 100644 index 00000000000..8a77647a956 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/file.go @@ -0,0 +1,1258 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Well known handles +// 1: root +// 2: id + +package ql + +import ( + "crypto/sha1" + "fmt" + "io" + "io/ioutil" + "log" + "math/big" + "os" + "path/filepath" + "sync" + "time" + + "github.com/camlistore/lock" + "github.com/cznic/exp/lldb" + "github.com/cznic/mathutil" +) + +const ( + magic = "\x60\xdbql" +) + +var ( + _ btreeIndex = (*fileIndex)(nil) + _ btreeIterator = (*fileBTreeIterator)(nil) + _ indexIterator = (*fileIndexIterator)(nil) + _ storage = (*file)(nil) + _ temp = (*fileTemp)(nil) +) + +type chunk struct { // expanded to blob types lazily + f *file + b []byte +} + +func (c chunk) expand() (v interface{}, err error) { + return c.f.loadChunks(c.b) +} + +func expand1(data interface{}, e error) (v interface{}, err error) { + if e != nil { + return nil, e + } + + c, ok := data.(chunk) + if !ok { + return data, nil + } + + return c.expand() +} + +func expand(data []interface{}) (err error) { + for i, v := range data { + if data[i], err = expand1(v, nil); err != nil { + return + } + } + return +} + +// OpenFile returns a DB backed by a named file. The back end limits the size +// of a record to about 64 kB. +func OpenFile(name string, opt *Options) (db *DB, err error) { + var f lldb.OSFile + if f = opt.OSFile; f == nil { + f, err = os.OpenFile(name, os.O_RDWR, 0666) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + + if !opt.CanCreate { + return nil, err + } + + f, err = os.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666) + if err != nil { + return nil, err + } + } + } + + fi, err := newFileFromOSFile(f) // always ACID + if err != nil { + return + } + + if fi.tempFile = opt.TempFile; fi.tempFile == nil { + fi.tempFile = func(dir, prefix string) (f lldb.OSFile, err error) { + f0, err := ioutil.TempFile(dir, prefix) + return f0, err + } + } + + return newDB(fi) +} + +// Options amend the behavior of OpenFile. +// +// CanCreate +// +// The CanCreate option enables OpenFile to create the DB file if it does not +// exists. +// +// OSFile +// +// OSFile allows to pass an os.File like back end providing, for example, +// encrypted storage. If this field is nil then OpenFile uses the file named by +// the 'name' parameter instead. +// +// TempFile +// +// TempFile provides a temporary file used for evaluating the GROUP BY, ORDER +// BY, ... clauses. The hook is intended to be used by encrypted DB back ends +// to avoid leaks of unecrypted data to such temp files by providing temp files +// which are encrypted as well. Note that *os.File satisfies the lldb.OSFile +// interface. +// +// If TempFile is nil it defaults to ioutil.TempFile. +type Options struct { + CanCreate bool + OSFile lldb.OSFile + TempFile func(dir, prefix string) (f lldb.OSFile, err error) +} + +type fileBTreeIterator struct { + en *lldb.BTreeEnumerator + t *fileTemp +} + +func (it *fileBTreeIterator) Next() (k, v []interface{}, err error) { + bk, bv, err := it.en.Next() + if err != nil { + return + } + + if k, err = lldb.DecodeScalars(bk); err != nil { + return + } + + for i, val := range k { + b, ok := val.([]byte) + if !ok { + continue + } + + c := chunk{it.t.file, b} + if k[i], err = c.expand(); err != nil { + return nil, nil, err + } + } + + if err = enforce(k, it.t.colsK); err != nil { + return + } + + if v, err = lldb.DecodeScalars(bv); err != nil { + return + } + + for i, val := range v { + b, ok := val.([]byte) + if !ok { + continue + } + + c := chunk{it.t.file, b} + if v[i], err = c.expand(); err != nil { + return nil, nil, err + } + } + + err = enforce(v, it.t.colsV) + return +} + +func enforce(val []interface{}, cols []*col) (err error) { + for i, v := range val { + if val[i], err = convert(v, cols[i].typ); err != nil { + return + } + } + return +} + +//NTYPE +func infer(from []interface{}, to *[]*col) { + if len(*to) == 0 { + *to = make([]*col, len(from)) + for i := range *to { + (*to)[i] = &col{} + } + } + for i, c := range *to { + if f := from[i]; f != nil { + switch x := f.(type) { + //case nil: + case idealComplex: + c.typ = qComplex128 + from[i] = complex128(x) + case idealFloat: + c.typ = qFloat64 + from[i] = float64(x) + case idealInt: + c.typ = qInt64 + from[i] = int64(x) + case idealRune: + c.typ = qInt32 + from[i] = int32(x) + case idealUint: + c.typ = qUint64 + from[i] = uint64(x) + case bool: + c.typ = qBool + case complex128: + c.typ = qComplex128 + case complex64: + c.typ = qComplex64 + case float64: + c.typ = qFloat64 + case float32: + c.typ = qFloat32 + case int8: + c.typ = qInt8 + case int16: + c.typ = qInt16 + case int32: + c.typ = qInt32 + case int64: + c.typ = qInt64 + case string: + c.typ = qString + case uint8: + c.typ = qUint8 + case uint16: + c.typ = qUint16 + case uint32: + c.typ = qUint32 + case uint64: + c.typ = qUint64 + case []byte: + c.typ = qBlob + case *big.Int: + c.typ = qBigInt + case *big.Rat: + c.typ = qBigRat + case time.Time: + c.typ = qTime + case time.Duration: + c.typ = qDuration + case chunk: + vals, err := lldb.DecodeScalars([]byte(x.b)) + if err != nil { + log.Panic("err") + } + + if len(vals) == 0 { + log.Panic("internal error 040") + } + + i, ok := vals[0].(int64) + if !ok { + log.Panic("internal error 041") + } + + c.typ = int(i) + case map[string]interface{}: // map of ids of a cross join + default: + log.Panic("internal error 042") + } + } + } +} + +type fileTemp struct { + *file + colsK []*col + colsV []*col + t *lldb.BTree +} + +func (t *fileTemp) BeginTransaction() error { + return nil +} + +func (t *fileTemp) Get(k []interface{}) (v []interface{}, err error) { + if err = expand(k); err != nil { + return + } + + if err = t.flatten(k); err != nil { + return nil, err + } + + bk, err := lldb.EncodeScalars(k...) + if err != nil { + return + } + + bv, err := t.t.Get(nil, bk) + if err != nil { + return + } + + return lldb.DecodeScalars(bv) +} + +func (t *fileTemp) Drop() (err error) { + if t.f0 == nil { + return + } + + fn := t.f0.Name() + if err = t.f0.Close(); err != nil { + return + } + + if fn == "" { + return + } + + return os.Remove(fn) +} + +func (t *fileTemp) SeekFirst() (it btreeIterator, err error) { + en, err := t.t.SeekFirst() + if err != nil { + return + } + + return &fileBTreeIterator{t: t, en: en}, nil +} + +func (t *fileTemp) Set(k, v []interface{}) (err error) { + if err = expand(k); err != nil { + return + } + + if err = expand(v); err != nil { + return + } + + infer(k, &t.colsK) + infer(v, &t.colsV) + + if err = t.flatten(k); err != nil { + return + } + + bk, err := lldb.EncodeScalars(k...) + if err != nil { + return + } + + if err = t.flatten(v); err != nil { + return + } + + bv, err := lldb.EncodeScalars(v...) + if err != nil { + return + } + + return t.t.Set(bk, bv) +} + +type file struct { + a *lldb.Allocator + codec *gobCoder + f lldb.Filer + f0 lldb.OSFile + id int64 + lck io.Closer + mu sync.Mutex + name string + tempFile func(dir, prefix string) (f lldb.OSFile, err error) + wal *os.File +} + +func newFileFromOSFile(f lldb.OSFile) (fi *file, err error) { + nm := lockName(f.Name()) + lck, err := lock.Lock(nm) + if err != nil { + if lck != nil { + lck.Close() + } + return nil, err + } + + close := true + defer func() { + if close && lck != nil { + lck.Close() + } + }() + + var w *os.File + closew := false + wn := walName(f.Name()) + w, err = os.OpenFile(wn, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666) + closew = true + defer func() { + if closew { + nm := w.Name() + w.Close() + os.Remove(nm) + w = nil + } + }() + + if err != nil { + if !os.IsExist(err) { + return nil, err + } + + closew = false + w, err = os.OpenFile(wn, os.O_RDWR, 0666) + if err != nil { + return nil, err + } + + closew = true + st, err := w.Stat() + if err != nil { + return nil, err + } + + if st.Size() != 0 { + return nil, fmt.Errorf("(file-001) non empty WAL file %s exists", wn) + } + } + + info, err := f.Stat() + if err != nil { + return nil, err + } + + switch sz := info.Size(); { + case sz == 0: + b := make([]byte, 16) + copy(b, []byte(magic)) + if _, err := f.Write(b); err != nil { + return nil, err + } + + filer := lldb.Filer(lldb.NewOSFiler(f)) + filer = lldb.NewInnerFiler(filer, 16) + if filer, err = lldb.NewACIDFiler(filer, w); err != nil { + return nil, err + } + + a, err := lldb.NewAllocator(filer, &lldb.Options{}) + if err != nil { + return nil, err + } + + a.Compress = true + s := &file{ + a: a, + codec: newGobCoder(), + f0: f, + f: filer, + lck: lck, + name: f.Name(), + wal: w, + } + if err = s.BeginTransaction(); err != nil { + return nil, err + } + + h, err := s.Create() + if err != nil { + return nil, err + } + + if h != 1 { // root + log.Panic("internal error 043") + } + + if h, err = s.a.Alloc(make([]byte, 8)); err != nil { + return nil, err + } + + if h != 2 { // id + log.Panic("internal error 044") + } + + close, closew = false, false + return s, s.Commit() + default: + b := make([]byte, 16) + if _, err := f.Read(b); err != nil { + return nil, err + } + + if string(b[:len(magic)]) != magic { + return nil, fmt.Errorf("(file-002) unknown file format") + } + + filer := lldb.Filer(lldb.NewOSFiler(f)) + filer = lldb.NewInnerFiler(filer, 16) + if filer, err = lldb.NewACIDFiler(filer, w); err != nil { + return nil, err + } + + a, err := lldb.NewAllocator(filer, &lldb.Options{}) + if err != nil { + return nil, err + } + + bid, err := a.Get(nil, 2) // id + if err != nil { + return nil, err + } + + if len(bid) != 8 { + return nil, fmt.Errorf("(file-003) corrupted DB: id |% x|", bid) + } + + id := int64(0) + for _, v := range bid { + id = (id << 8) | int64(v) + } + + a.Compress = true + s := &file{ + a: a, + codec: newGobCoder(), + f0: f, + f: filer, + id: id, + lck: lck, + name: f.Name(), + wal: w, + } + + close, closew = false, false + return s, nil + } +} + +func (s *file) OpenIndex(unique bool, handle int64) (btreeIndex, error) { + t, err := lldb.OpenBTree(s.a, s.collate, handle) + if err != nil { + return nil, err + } + + return &fileIndex{s, handle, t, unique}, nil +} + +func (s *file) CreateIndex(unique bool) ( /* handle */ int64, btreeIndex, error) { + t, h, err := lldb.CreateBTree(s.a, s.collate) + if err != nil { + return -1, nil, err + } + + return h, &fileIndex{s, h, t, unique}, nil +} + +func (s *file) Acid() bool { return s.wal != nil } + +func errSet(p *error, errs ...error) (err error) { + err = *p + for _, e := range errs { + if err != nil { + return + } + *p, err = e, e + } + return +} + +func (s *file) lock() func() { + s.mu.Lock() + return s.mu.Unlock +} + +func (s *file) Close() (err error) { + defer s.lock()() + + es := s.f0.Sync() + ef := s.f0.Close() + var ew error + if s.wal != nil { + ew = s.wal.Close() + } + el := s.lck.Close() + return errSet(&err, es, ef, ew, el) +} + +func (s *file) Name() string { return s.name } + +func (s *file) Verify() (allocs int64, err error) { + defer s.lock()() + var stat lldb.AllocStats + if err = s.a.Verify(lldb.NewMemFiler(), nil, &stat); err != nil { + return + } + + allocs = stat.AllocAtoms + return +} + +func (s *file) expandBytes(d []interface{}) (err error) { + for i, v := range d { + b, ok := v.([]byte) + if !ok { + continue + } + + d[i], err = s.loadChunks(b) + if err != nil { + return + } + } + return +} + +func (s *file) collate(a, b []byte) int { //TODO w/ error return + da, err := lldb.DecodeScalars(a) + if err != nil { + log.Panic(err) + } + + if err = s.expandBytes(da); err != nil { + log.Panic(err) + } + + db, err := lldb.DecodeScalars(b) + if err != nil { + log.Panic(err) + } + + if err = s.expandBytes(db); err != nil { + log.Panic(err) + } + + return collate(da, db) +} + +func (s *file) CreateTemp(asc bool) (bt temp, err error) { + f, err := s.tempFile("", "ql-tmp-") + if err != nil { + return nil, err + } + + fn := f.Name() + filer := lldb.NewOSFiler(f) + a, err := lldb.NewAllocator(filer, &lldb.Options{}) + if err != nil { + f.Close() + os.Remove(fn) + return nil, err + } + + k := 1 + if !asc { + k = -1 + } + + t, _, err := lldb.CreateBTree(a, func(a, b []byte) int { //TODO w/ error return + return k * s.collate(a, b) + }) + if err != nil { + f.Close() + if fn != "" { + os.Remove(fn) + } + return nil, err + } + + x := &fileTemp{file: &file{ + a: a, + codec: newGobCoder(), + f0: f, + }, + t: t} + return x, nil +} + +func (s *file) BeginTransaction() (err error) { + defer s.lock()() + return s.f.BeginUpdate() +} + +func (s *file) Rollback() (err error) { + defer s.lock()() + return s.f.Rollback() +} + +func (s *file) Commit() (err error) { + defer s.lock()() + return s.f.EndUpdate() +} + +func (s *file) Create(data ...interface{}) (h int64, err error) { + if err = expand(data); err != nil { + return + } + + if err = s.flatten(data); err != nil { + return + } + + b, err := lldb.EncodeScalars(data...) + if err != nil { + return + } + + defer s.lock()() + return s.a.Alloc(b) +} + +func (s *file) Delete(h int64, blobCols ...*col) (err error) { + switch len(blobCols) { + case 0: + defer s.lock()() + return s.a.Free(h) + default: + return s.free(h, blobCols) + } +} + +func (s *file) ResetID() (err error) { + s.id = 0 + return +} + +func (s *file) ID() (int64, error) { + defer s.lock()() + + s.id++ + b := make([]byte, 8) + id := s.id + for i := 7; i >= 0; i-- { + b[i] = byte(id) + id >>= 8 + } + + return s.id, s.a.Realloc(2, b) +} + +func (s *file) free(h int64, blobCols []*col) (err error) { + s.mu.Lock() + b, err := s.a.Get(nil, h) //LATER +bufs + s.mu.Unlock() + if err != nil { + return + } + + rec, err := lldb.DecodeScalars(b) + if err != nil { + return + } + + for _, col := range blobCols { + if col.index >= len(rec) { + return fmt.Errorf("(file-004) file.free: corrupted DB (record len)") + } + + switch x := rec[col.index+2].(type) { + case nil: + // nop + case []byte: + if err = s.freeChunks(x); err != nil { + return + } + } + } + defer s.lock()() + return s.a.Free(h) +} + +func (s *file) Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) { //NTYPE + s.mu.Lock() + b, err := s.a.Get(nil, h) //LATER +bufs + s.mu.Unlock() + if err != nil { + return + } + + rec, err := lldb.DecodeScalars(b) + if err != nil { + return + } + + for _, col := range cols { + i := col.index + 2 + switch col.typ { + case 0: + case qBool: + case qComplex64: + rec[i] = complex64(rec[i].(complex128)) + case qComplex128: + case qFloat32: + rec[i] = float32(rec[i].(float64)) + case qFloat64: + case qInt8: + rec[i] = int8(rec[i].(int64)) + case qInt16: + rec[i] = int16(rec[i].(int64)) + case qInt32: + rec[i] = int32(rec[i].(int64)) + case qInt64: + case qString: + case qUint8: + rec[i] = uint8(rec[i].(uint64)) + case qUint16: + rec[i] = uint16(rec[i].(uint64)) + case qUint32: + rec[i] = uint32(rec[i].(uint64)) + case qUint64: + case qBlob, qBigInt, qBigRat, qTime, qDuration: + switch x := rec[i].(type) { + case nil: + rec[i] = nil + case []byte: + rec[i] = chunk{f: s, b: x} + default: + return nil, fmt.Errorf("(file-006) corrupted DB: non nil chunk type is not []byte") + } + default: + log.Panic("internal error 045") + } + } + + if cols != nil { + for n, dn := len(cols)+2, len(rec); dn < n; dn++ { + rec = append(rec, nil) + } + } + + return rec, nil +} + +func (s *file) freeChunks(enc []byte) (err error) { + items, err := lldb.DecodeScalars(enc) + if err != nil { + return + } + + var ok bool + var next int64 + switch len(items) { + case 2: + return + case 3: + if next, ok = items[1].(int64); !ok || next == 0 { + return fmt.Errorf("(file-007) corrupted DB: first chunk link") + } + default: + return fmt.Errorf("(file-008) corrupted DB: first chunk") + } + + for next != 0 { + s.mu.Lock() + b, err := s.a.Get(nil, next) + s.mu.Unlock() + if err != nil { + return err + } + + if items, err = lldb.DecodeScalars(b); err != nil { + return err + } + + var h int64 + switch len(items) { + case 1: + // nop + case 2: + if h, ok = items[0].(int64); !ok { + return fmt.Errorf("(file-009) corrupted DB: chunk link") + } + + default: + return fmt.Errorf("(file-010) corrupted DB: chunk items %d (%v)", len(items), items) + } + + s.mu.Lock() + if err = s.a.Free(next); err != nil { + s.mu.Unlock() + return err + } + + s.mu.Unlock() + next = h + } + return +} + +func (s *file) loadChunks(enc []byte) (v interface{}, err error) { + items, err := lldb.DecodeScalars(enc) + if err != nil { + return + } + + var ok bool + var next int64 + switch len(items) { + case 2: + // nop + case 3: + if next, ok = items[1].(int64); !ok || next == 0 { + return nil, fmt.Errorf("(file-011) corrupted DB: first chunk link") + } + default: + //fmt.Printf("%d: %#v\n", len(items), items) + return nil, fmt.Errorf("(file-012) corrupted DB: first chunk") + } + + typ, ok := items[0].(int64) + if !ok { + return nil, fmt.Errorf("(file-013) corrupted DB: first chunk tag") + } + + buf, ok := items[len(items)-1].([]byte) + if !ok { + return nil, fmt.Errorf("(file-014) corrupted DB: first chunk data") + } + + for next != 0 { + s.mu.Lock() + b, err := s.a.Get(nil, next) + s.mu.Unlock() + if err != nil { + return nil, err + } + + if items, err = lldb.DecodeScalars(b); err != nil { + return nil, err + } + + switch len(items) { + case 1: + next = 0 + case 2: + if next, ok = items[0].(int64); !ok { + return nil, fmt.Errorf("(file-015) corrupted DB: chunk link") + } + + items = items[1:] + default: + return nil, fmt.Errorf("(file-016) corrupted DB: chunk items %d (%v)", len(items), items) + } + + if b, ok = items[0].([]byte); !ok { + return nil, fmt.Errorf("(file-017) corrupted DB: chunk data") + } + + buf = append(buf, b...) + } + return s.codec.decode(buf, int(typ)) +} + +func (s *file) Update(h int64, data ...interface{}) (err error) { + b, err := lldb.EncodeScalars(data...) + if err != nil { + return + } + + defer s.lock()() + return s.a.Realloc(h, b) +} + +func (s *file) UpdateRow(h int64, blobCols []*col, data ...interface{}) (err error) { + if len(blobCols) == 0 { + return s.Update(h, data...) + } + + if err = expand(data); err != nil { + return + } + + data0, err := s.Read(nil, h, blobCols...) + if err != nil { + return + } + + for _, c := range blobCols { + if err = s.freeChunks(data0[c.index+2].(chunk).b); err != nil { + return + } + } + + if err = s.flatten(data); err != nil { + return + } + + return s.Update(h, data...) +} + +// []interface{}{qltype, ...}->[]interface{}{lldb scalar type, ...} +// + long blobs are (pre)written to a chain of chunks. +func (s *file) flatten(data []interface{}) (err error) { + for i, v := range data { + tag := 0 + var b []byte + switch x := v.(type) { + case []byte: + tag = qBlob + b = x + case *big.Int: + tag = qBigInt + b, err = s.codec.encode(x) + case *big.Rat: + tag = qBigRat + b, err = s.codec.encode(x) + case time.Time: + tag = qTime + b, err = s.codec.encode(x) + case time.Duration: + tag = qDuration + b, err = s.codec.encode(x) + default: + continue + } + if err != nil { + return + } + + const chunk = 1 << 16 + chunks := 0 + var next int64 + var buf []byte + for rem := len(b); rem > shortBlob; { + n := mathutil.Min(rem, chunk) + part := b[rem-n:] + b = b[:rem-n] + rem -= n + switch next { + case 0: // last chunk + buf, err = lldb.EncodeScalars([]interface{}{part}...) + default: // middle chunk + buf, err = lldb.EncodeScalars([]interface{}{next, part}...) + } + if err != nil { + return + } + + s.mu.Lock() + h, err := s.a.Alloc(buf) + s.mu.Unlock() + if err != nil { + return err + } + + next = h + chunks++ + } + + switch next { + case 0: // single chunk + buf, err = lldb.EncodeScalars([]interface{}{tag, b}...) + default: // multi chunks + buf, err = lldb.EncodeScalars([]interface{}{tag, next, b}...) + } + if err != nil { + return + } + + data[i] = buf + } + return +} + +func lockName(dbname string) string { + base := filepath.Base(filepath.Clean(dbname)) + "lockfile" + h := sha1.New() + io.WriteString(h, base) + return filepath.Join(filepath.Dir(dbname), fmt.Sprintf(".%x", h.Sum(nil))) +} + +func walName(dbname string) (r string) { + base := filepath.Base(filepath.Clean(dbname)) + h := sha1.New() + io.WriteString(h, base) + return filepath.Join(filepath.Dir(dbname), fmt.Sprintf(".%x", h.Sum(nil))) +} + +type fileIndex struct { + f *file + h int64 + t *lldb.BTree + unique bool +} + +func (x *fileIndex) Clear() error { + return x.t.Clear() +} + +var gbZeroInt64 []byte + +func init() { + var err error + if gbZeroInt64, err = lldb.EncodeScalars(int64(0)); err != nil { + panic(err) + } +} + +// The []byte version of the key in the BTree shares chunks, if any, with +// the value stored in the record. +func (x *fileIndex) Create(indexedValue interface{}, h int64) error { + t := x.t + switch { + case !x.unique: + k, err := lldb.EncodeScalars(indexedValue, h) + if err != nil { + return err + } + + return t.Set(k, gbZeroInt64) + case indexedValue == nil: // unique, NULL + k, err := lldb.EncodeScalars(nil, h) + if err != nil { + return err + } + + return t.Set(k, gbZeroInt64) + default: // unique, non NULL + k, err := lldb.EncodeScalars(indexedValue, int64(0)) + if err != nil { + return err + } + + v, err := lldb.EncodeScalars(h) + if err != nil { + return err + } + + _, _, err = t.Put(nil, k, func(key, old []byte) (new []byte, write bool, err error) { + if old == nil { + return v, true, nil + } + + return nil, false, fmt.Errorf("(file-018) cannot insert into unique index: duplicate value: %v", indexedValue) + }) + return err + } +} + +func (x *fileIndex) Delete(indexedValue interface{}, h int64) error { + chunk, ok := indexedValue.(chunk) + if ok { + indexedValue = chunk.b + } + + t := x.t + var k []byte + var err error + switch { + case !x.unique: + k, err = lldb.EncodeScalars(indexedValue, h) + case indexedValue == nil: // unique, NULL + k, err = lldb.EncodeScalars(nil, h) + default: // unique, non NULL + k, err = lldb.EncodeScalars(indexedValue, int64(0)) + } + if err != nil { + return err + } + + return t.Delete(k) +} + +func (x *fileIndex) Drop() error { + if err := x.Clear(); err != nil { + return err + } + + return x.f.a.Free(x.h) +} + +func (x *fileIndex) Seek(indexedValue interface{}) (indexIterator, bool, error) { //TODO(indices) blobs: +test + k, err := lldb.EncodeScalars(indexedValue, 0) + if err != nil { + return nil, false, err + } + + en, hit, err := x.t.Seek(k) + if err != nil { + return nil, false, err + } + + return &fileIndexIterator{x.f, en, x.unique}, hit, nil +} + +func (x *fileIndex) SeekFirst() (iter indexIterator, err error) { + en, err := x.t.SeekFirst() + return &fileIndexIterator{x.f, en, x.unique}, err +} + +func (x *fileIndex) SeekLast() (iter indexIterator, err error) { + en, err := x.t.SeekLast() + return &fileIndexIterator{x.f, en, x.unique}, err +} + +type fileIndexIterator struct { + f *file + en *lldb.BTreeEnumerator + unique bool +} + +func (i *fileIndexIterator) nextPrev(f func() ([]byte, []byte, error)) (interface{}, int64, error) { //TODO(indices) blobs: +test + bk, bv, err := f() + if err != nil { + return nil, -1, err + } + + dk, err := lldb.DecodeScalars(bk) + if err != nil { + return nil, -1, err + } + + b, ok := dk[0].([]byte) + if ok { + dk[0] = chunk{i.f, b} + if expand(dk[:1]); err != nil { + return nil, -1, err + } + } + + var k indexKey + k.value = dk[0] + switch i.unique { + case true: + if k.value == nil { + return nil, dk[1].(int64), nil + } + + dv, err := lldb.DecodeScalars(bv) + if err != nil { + return nil, -1, err + } + + return k.value, dv[0].(int64), nil + default: + return k.value, dk[1].(int64), nil + } +} + +func (i *fileIndexIterator) Next() (interface{}, int64, error) { //TODO(indices) blobs: +test + return i.nextPrev(i.en.Next) +} + +func (i *fileIndexIterator) Prev() (interface{}, int64, error) { //TODO(indices) blobs: +test + return i.nextPrev(i.en.Prev) +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/helper.go b/Godeps/_workspace/src/github.com/cznic/ql/helper.go new file mode 100644 index 00000000000..a21a4615661 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/helper.go @@ -0,0 +1,338 @@ +// +build ignore + +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "log" + "os" +) + +type t int + +const ( + qNil t = iota + idealComplex + idealFloat + idealInt + idealRune + idealUint + qBool + qComplex64 + qComplex128 + qFloat32 + qFloat64 + qInt8 + qInt16 + qInt32 + qInt64 + qString + qUint8 + qUint16 + qUint32 + qUint64 + qBigInt + qBigRat + qTime + qDuration + + qEnd +) + +func (n t) String() string { + switch n { + case qNil: + return "nil" + case idealComplex: + return "idealComplex" + case idealFloat: + return "idealFloat" + case idealInt: + return "idealInt" + case idealRune: + return "idealRune" + case idealUint: + return "idealUint" + case qBool: + return "bool" + case qComplex64: + return "complex64" + case qComplex128: + return "complex128" + case qFloat32: + return "float32" + case qFloat64: + return "float64" + case qInt8: + return "int8" + case qInt16: + return "int16" + case qInt32: + return "int32" + case qInt64: + return "int64" + case qString: + return "string" + case qUint8: + return "uint8" + case qUint16: + return "uint16" + case qUint32: + return "uint32" + case qUint64: + return "uint64" + case qBigInt: + return "*big.Int" + case qBigRat: + return "*big.Rat" + case qTime: + return "time.Time" + case qDuration: + return "time.Duration" + default: + panic("internal error 046") + } +} + +func coerceIdealComplex(typ t) string { + switch typ { + case qComplex64, qComplex128: + return fmt.Sprintf("return %s(x)\n", typ) + default: + return "" + } +} + +func coerceIdealFloat(typ t) string { + switch typ { + case idealComplex: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case qComplex64: + return fmt.Sprintf("return %s(complex(float32(x), 0))\n", typ) + case qComplex128: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case idealFloat, qFloat32, qFloat64: + return fmt.Sprintf("return %s(float64(x))\n", typ) + case qBigRat: + return fmt.Sprintf("return big.NewRat(1, 1).SetFloat64(float64(x))\n") + default: + return "" + } + return "" +} + +func coerceIdealInt(typ t) string { + switch typ { + case idealComplex: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case qComplex64: + return fmt.Sprintf("return %s(complex(float32(x), 0))\n", typ) + case qComplex128: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case idealFloat, idealInt, qFloat32, qFloat64, qInt64: + return fmt.Sprintf("return %s(int64(x))\n", typ) + case idealUint: + return fmt.Sprintf("if x >= 0 { return %s(int64(x)) }\n", typ) + case qInt8: + return fmt.Sprintf("if x >= math.MinInt8 && x<= math.MaxInt8 { return %s(int64(x)) }\n", typ) + case qInt16: + return fmt.Sprintf("if x >= math.MinInt16 && x<= math.MaxInt16 { return %s(int64(x)) }\n", typ) + case qInt32: + return fmt.Sprintf("if x >= math.MinInt32 && x<= math.MaxInt32 { return %s(int64(x)) }\n", typ) + case qUint8: + return fmt.Sprintf("if x >= 0 && x<= math.MaxUint8 { return %s(int64(x)) }\n", typ) + case qUint16: + return fmt.Sprintf("if x >= 0 && x<= math.MaxUint16 { return %s(int64(x)) }\n", typ) + case qUint32: + return fmt.Sprintf("if x >= 0 && x<= math.MaxUint32 { return %s(int64(x)) }\n", typ) + case qUint64: + return fmt.Sprintf("if x >= 0 { return %s(int64(x)) }\n", typ) + case qBigInt: + return fmt.Sprintf("return big.NewInt(int64(x))\n") + case qBigRat: + return fmt.Sprintf("return big.NewRat(1, 1).SetInt64(int64(x))\n") + case qDuration: + return fmt.Sprintf("return time.Duration(int64(x))\n") + default: + return "" + } + return "" +} + +func coerceIdealRune(typ t) string { + switch typ { + case idealComplex: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case qComplex64: + return fmt.Sprintf("return %s(complex(float32(x), 0))\n", typ) + case qComplex128: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case idealFloat, idealInt, idealRune, idealUint, qFloat32, qFloat64, qInt8, qInt16, qInt32, qInt64, qUint8, qUint16, qUint32, qUint64: + return fmt.Sprintf("return %s(int64(x))\n", typ) + case qBigInt: + return fmt.Sprintf("return big.NewInt(int64(x))\n") + case qBigRat: + return fmt.Sprintf("return big.NewRat(1, 1).SetInt64(int64(x))\n") + case qDuration: + return fmt.Sprintf("return time.Duration(int64(x))\n") + default: + return "" + } + return "" +} + +func coerceIdealUint(typ t) string { + switch typ { + case idealComplex: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case qComplex64: + return fmt.Sprintf("return %s(complex(float32(x), 0))\n", typ) + case qComplex128: + return fmt.Sprintf("return %s(complex(float64(x), 0))\n", typ) + case idealFloat, idealUint, qFloat32, qFloat64, qUint64: + return fmt.Sprintf("return %s(uint64(x))\n", typ) + case idealInt: + return fmt.Sprintf("if x <= math.MaxInt64 { return %s(int64(x)) }\n", typ) + case qInt8: + return fmt.Sprintf("if x <= math.MaxInt8 { return %s(int64(x)) }\n", typ) + case qInt16: + return fmt.Sprintf("if x<= math.MaxInt16 { return %s(int64(x)) }\n", typ) + case qInt32: + return fmt.Sprintf("if x<= math.MaxInt32 { return %s(int64(x)) }\n", typ) + case qInt64: + return fmt.Sprintf("if x<= math.MaxInt64 { return %s(int64(x)) }\n", typ) + case qUint8: + return fmt.Sprintf("if x >= 0 && x<= math.MaxUint8 { return %s(int64(x)) }\n", typ) + case qUint16: + return fmt.Sprintf("if x >= 0 && x<= math.MaxUint16 { return %s(int64(x)) }\n", typ) + case qUint32: + return fmt.Sprintf("if x >= 0 && x<= math.MaxUint32 { return %s(int64(x)) }\n", typ) + case qBigInt: + return fmt.Sprintf("return big.NewInt(0).SetUint64(uint64(x))\n") + case qBigRat: + return fmt.Sprintf("return big.NewRat(1, 1).SetInt(big.NewInt(0).SetUint64(uint64(x)))\n") + case qDuration: + return fmt.Sprintf("if x <= math.MaxInt64 { return time.Duration(int64(x)) }\n") + default: + return "" + } + return "" +} + +func genCoerce1(w io.Writer, in t, f func(out t) string) { + fmt.Fprintf(w, "\tcase %s:\n", in) + fmt.Fprintf(w, "\t\tswitch otherVal.(type) {\n") + + for i := idealComplex; i < qEnd; i++ { + s := f(i) + switch s { + case "": + fmt.Fprintf(w, "\t\t//case %s:\n", i) + default: + fmt.Fprintf(w, "\t\tcase %s:\n", i) + fmt.Fprintf(w, "\t\t\t%s", s) + } + } + + fmt.Fprintf(w, "\t\t}\n") // switch +} + +func genCoerce(w io.Writer) { + fmt.Fprintf(w, + ` +func coerce1(inVal, otherVal interface{}) (coercedInVal interface{}) { + coercedInVal = inVal + if otherVal == nil { + return + } + + switch x := inVal.(type) { + case nil: + return +`) + genCoerce1(w, idealComplex, coerceIdealComplex) + genCoerce1(w, idealFloat, coerceIdealFloat) + genCoerce1(w, idealInt, coerceIdealInt) + genCoerce1(w, idealRune, coerceIdealRune) + genCoerce1(w, idealUint, coerceIdealUint) + fmt.Fprintf(w, "\t}\n") // switch + + fmt.Fprintf(w, "\treturn\n}\n") // func +} + +func main() { + ofn := flag.String("o", "", "") + flag.Parse() + _, err := os.Stat(*ofn) + if err == nil { + log.Fatalf("%s exists", *ofn) + } + + w := bufio.NewWriter(os.Stdout) + if s := *ofn; s != "" { + f, err := os.Create(s) + if err != nil { + log.Fatal(err) + } + + defer f.Close() + w = bufio.NewWriter(f) + } + defer w.Flush() + + fmt.Fprintf(w, `// Copyright 2013 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: This file was generated automatically by +// +// $ go run helper -o coerce.go +// +// DO NOT EDIT! + + package ql + +import ( + "math" + "math/big" + "reflect" + "time" +) + +func coerce(a, b interface{}) (x, y interface{}) { + if reflect.TypeOf(a) == reflect.TypeOf(b) { + return a, b + } + + switch a.(type) { + case idealComplex, idealFloat, idealInt, idealRune, idealUint: + switch b.(type) { + case idealComplex, idealFloat, idealInt, idealRune, idealUint: + x, y = coerce1(a, b), b + if reflect.TypeOf(x) == reflect.TypeOf(y) { + return + } + + return a, coerce1(b, a) + default: + return coerce1(a, b), b + } + default: + switch b.(type) { + case idealComplex, idealFloat, idealInt, idealRune, idealUint: + return a, coerce1(b, a) + default: + return a, b + } + } +} +`) + genCoerce(w) +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/httpfs.go b/Godeps/_workspace/src/github.com/cznic/ql/httpfs.go new file mode 100644 index 00000000000..89c6f89c52c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/httpfs.go @@ -0,0 +1,302 @@ +// Copyright (c) 2014 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/cznic/mathutil" +) + +var ( + _ http.FileSystem = (*HTTPFS)(nil) + _ http.File = (*HTTPFile)(nil) + _ os.FileInfo = (*HTTPFile)(nil) + _ os.FileInfo = (*dirEntry)(nil) +) + +type dirEntry string + +func (d dirEntry) Name() string { return string(d) } +func (d dirEntry) Size() int64 { return -1 } +func (d dirEntry) Mode() os.FileMode { return os.ModeDir } +func (d dirEntry) ModTime() time.Time { return time.Time{} } +func (d dirEntry) IsDir() bool { return true } +func (d dirEntry) Sys() interface{} { return interface{}(nil) } + +// A HTTPFile is returned by the HTTPFS's Open method and can be served by the +// http.FileServer implementation. +type HTTPFile struct { + closed bool + content []byte + dirEntries []os.FileInfo + isFile bool + name string + off int + sz int +} + +// Close implements http.File. +func (f *HTTPFile) Close() error { + if f.closed { + return os.ErrInvalid + } + + f.closed = true + return nil +} + +// IsDir implements os.FileInfo +func (f *HTTPFile) IsDir() bool { return !f.isFile } + +// Mode implements os.FileInfo +func (f *HTTPFile) Mode() os.FileMode { + switch f.isFile { + case false: + return os.FileMode(0444) + default: + return os.ModeDir + } +} + +// ModTime implements os.FileInfo +func (f *HTTPFile) ModTime() time.Time { + return time.Time{} +} + +// Name implements os.FileInfo +func (f *HTTPFile) Name() string { return path.Base(f.name) } + +// Size implements os.FileInfo +func (f *HTTPFile) Size() int64 { + switch f.isFile { + case false: + return -1 + default: + return int64(len(f.content)) + } +} + +// Stat implements http.File. +func (f *HTTPFile) Stat() (os.FileInfo, error) { return f, nil } + +// Sys implements os.FileInfo +func (f *HTTPFile) Sys() interface{} { return interface{}(nil) } + +// Readdir implements http.File. +func (f *HTTPFile) Readdir(count int) ([]os.FileInfo, error) { + if f.isFile { + return nil, fmt.Errorf("not a directory: %s", f.name) + } + + if count <= 0 { + r := f.dirEntries + f.dirEntries = f.dirEntries[:0] + return r, nil + } + + rq := mathutil.Min(count, len(f.dirEntries)) + r := f.dirEntries[:rq] + f.dirEntries = f.dirEntries[rq:] + if len(r) != 0 { + return r, nil + } + + return nil, io.EOF +} + +// Read implements http.File. +func (f *HTTPFile) Read(b []byte) (int, error) { + if f.closed { + return 0, os.ErrInvalid + } + + n := copy(b, f.content[f.off:]) + f.off += n + if n != 0 { + return n, nil + } + + return 0, io.EOF +} + +// Seek implements http.File. +func (f *HTTPFile) Seek(offset int64, whence int) (int64, error) { + if f.closed { + return 0, os.ErrInvalid + } + + if offset < 0 { + return int64(f.off), fmt.Errorf("cannot seek before start of file") + } + + switch whence { + case 0: + noff := int64(f.off) + offset + if noff > mathutil.MaxInt { + return int64(f.off), fmt.Errorf("seek target overflows int: %d", noff) + } + + f.off = mathutil.Min(int(offset), len(f.content)) + if f.off == int(offset) { + return offset, nil + } + + return int64(f.off), io.EOF + case 1: + noff := int64(f.off) + offset + if noff > mathutil.MaxInt { + return int64(f.off), fmt.Errorf("seek target overflows int: %d", noff) + } + + off := mathutil.Min(f.off+int(offset), len(f.content)) + if off == f.off+int(offset) { + f.off = off + return int64(off), nil + } + + f.off = off + return int64(off), io.EOF + case 2: + noff := int64(f.off) - offset + if noff < 0 { + return int64(f.off), fmt.Errorf("cannot seek before start of file") + } + + f.off = len(f.content) - int(offset) + return int64(f.off), nil + default: + return int64(f.off), fmt.Errorf("seek: invalid whence %d", whence) + } +} + +// HTTPFS implements a http.FileSystem backed by data in a DB. +type HTTPFS struct { + db *DB + dir, get List +} + +// NewHTTPFS returns a http.FileSystem backed by a result record set of query. +// The record set provides two mandatory fields: path and content (the field +// names are case sensitive). Type of name must be string and type of content +// must be blob (ie. []byte). Field 'path' value is the "file" pathname, which +// must be rooted; and field 'content' value is its "data". +func (db *DB) NewHTTPFS(query string) (*HTTPFS, error) { + if _, err := Compile(query); err != nil { + return nil, err + } + + dir, err := Compile(fmt.Sprintf("SELECT path FROM (%s) WHERE hasPrefix(path, $1)", query)) + if err != nil { + return nil, err + } + + get, err := Compile(fmt.Sprintf("SELECT content FROM (%s) WHERE path == $1", query)) + if err != nil { + return nil, err + } + + return &HTTPFS{db: db, dir: dir, get: get}, nil +} + +// Open implements http.FileSystem. The name parameter represents a file path. +// The elements in a file path are separated by slash ('/', U+002F) characters, +// regardless of host operating system convention. +func (f *HTTPFS) Open(name string) (http.File, error) { + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return nil, fmt.Errorf("invalid character in file path: %q", name) + } + + name = path.Clean("/" + name) + rs, _, err := f.db.Execute(nil, f.get, name) + if err != nil { + return nil, err + } + + n := 0 + var fdata []byte + if err = rs[0].Do(false, func(data []interface{}) (more bool, err error) { + switch n { + case 0: + var ok bool + fdata, ok = data[0].([]byte) + if !ok { + return false, fmt.Errorf("open: expected blob, got %T", data[0]) + } + n++ + return true, nil + default: + return false, fmt.Errorf("open: more than one result was returned for %s", name) + } + }); err != nil { + return nil, err + } + + if n == 1 { // file found + return &HTTPFile{name: name, isFile: true, content: fdata}, nil + } + + dirName := name + if dirName[len(dirName)-1] != filepath.Separator { + dirName += string(filepath.Separator) + } + // Open("/a/b"): {/a/b/c.x,/a/b/d.x,/a/e.x,/a/b/f/g.x} -> {c.x,d.x,f} + rs, _, err = f.db.Execute(nil, f.dir, dirName) + if err != nil { + return nil, err + } + + n = 0 + r := &HTTPFile{name: dirName} + m := map[string]bool{} + x := len(dirName) + if err = rs[0].Do(false, func(data []interface{}) (more bool, err error) { + n++ + switch name := data[0].(type) { + case string: + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return false, fmt.Errorf("invalid character in file path: %q", name) + } + + name = path.Clean("/" + name) + rest := name[x:] + parts := strings.Split(rest, "/") + if len(parts) == 0 { + return true, nil + } + + nm := parts[0] + switch len(parts) { + case 1: // file + r.dirEntries = append(r.dirEntries, &HTTPFile{isFile: true, name: nm}) + default: // directory + if !m[nm] { + r.dirEntries = append(r.dirEntries, dirEntry(nm)) + } + m[nm] = true + } + return true, nil + default: + return false, fmt.Errorf("expected string path, got %T(%v)", name, name) + } + }); err != nil { + return nil, err + } + + if n != 0 { + return r, nil + } + + return nil, os.ErrNotExist +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/httpfs_test.go b/Godeps/_workspace/src/github.com/cznic/ql/httpfs_test.go new file mode 100644 index 00000000000..1ae79d305cc --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/httpfs_test.go @@ -0,0 +1,547 @@ +// Copyright (c) 2014 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "io" + "os" + "path" + "testing" +) + +func TestHTTP(t *testing.T) { + db, err := OpenMem() + if err != nil { + t.Fatal(err) + } + if _, _, err = db.Run( + NewRWCtx(), + ` + BEGIN TRANSACTION; + CREATE TABLE t (path string); + INSERT INTO t VALUES + ("/a/b/c/1"), + ("/a/b/c/2"), + ("/a/b/3"), + ("/a/b/4"), + ("/a/5"), + ("/a/6"), + ; + COMMIT; + `, + ); err != nil { + t.Fatal(err) + } + + fs, err := db.NewHTTPFS("SELECT path, blob(path+`-c`) AS content FROM t") + if err != nil { + t.Fatal(err) + } + + for _, nm := range []string{"/a/b/c/1", "/a/b/4", "/a/5"} { + f, err := fs.Open(nm) + if err != nil { + t.Fatal(err) + } + + stat, err := f.Stat() + if err != nil { + t.Fatal(err) + } + + b := make([]byte, 100) + n, err := f.Read(b) + if err != nil { + t.Fatal(nm, n, err) + } + + g := string(b[:n]) + if e := nm + "-c"; g != e { + t.Fatal(g, e) + } + + if g, e := stat.Name(), path.Base(nm); g != e { + t.Fatal(g, e) + } + + if g, e := stat.Size(), int64(len(g)); g != e { + t.Fatal(g, e) + } + + if g, e := stat.IsDir(), false; g != e { + t.Fatal(g, e) + } + + b = make([]byte, 100) + n, err = f.Read(b) + if n != 0 { + t.Error(n) + } + + if err != io.EOF { + t.Fatal(err) + } + + if n, err := f.Seek(0, 0); err != nil || n != 0 { + t.Fatal(n, err) + } + + exp := []byte(nm + "-c") + b = make([]byte, 1) + for _, e := range exp { + n, err := f.Read(b) + if n != 1 || err != nil { + t.Fatal(n, err) + } + + if g := b[0]; g != e { + t.Fatal(g, e) + } + } + if n, err := f.Read(b); n != 0 || err != io.EOF { + t.Fatal(n, err) + } + + if err = f.Close(); err != nil { + t.Fatal(err) + } + + if n, err := f.Seek(0, 0); err != os.ErrInvalid { + t.Fatal(n, err) + } + } + + if _, err = fs.Open("nonexistent"); err != os.ErrNotExist { + t.Fatal(err) + } + + // ------------------------------------------------------------------ / + d, err := fs.Open("") + if err != nil { + t.Fatal(err) + } + + stat, err := d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err := d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 1; g != e { + t.Fatal(g, e) + } + + var a bool + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "a": + a = true + default: + t.Fatal(v.Name()) + } + default: + t.Fatal(v.IsDir()) + } + } + if !a { + t.Fatal(a) + } + + // ------------------------------------------------------------------ a + d, err = fs.Open("a") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + var aB, a5, a6 bool + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "b": + aB = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "5": + a5 = true + case "6": + a6 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB && a5 && a6) { + t.Fatal(aB, a5, a6) + } + + // ----------------------------------------------------------------- a/ + d, err = fs.Open("a/") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "b": + aB = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "5": + a5 = true + case "6": + a6 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB && a5 && a6) { + t.Fatal(aB, a5, a6) + } + + // ----------------------------------------------------------------- /a + d, err = fs.Open("/a") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "b": + aB = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "5": + a5 = true + case "6": + a6 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB && a5 && a6) { + t.Fatal(aB, a5, a6) + } + + // ---------------------------------------------------------------- /a/ + d, err = fs.Open("/a/") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "b": + aB = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "5": + a5 = true + case "6": + a6 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB && a5 && a6) { + t.Fatal(aB, a5, a6) + } + + // ---------------------------------------------------------------- a/b + d, err = fs.Open("a/b") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + var aB3, aB4, aBC bool + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "c": + aBC = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "3": + aB3 = true + case "4": + aB4 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB3 && aB4 && aBC) { + t.Fatal(aB4, aB4, aBC) + } + + // --------------------------------------------------------------- a/b/ + d, err = fs.Open("a/b/") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "c": + aBC = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "3": + aB3 = true + case "4": + aB4 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB3 && aB4 && aBC) { + t.Fatal(aB4, aB4, aBC) + } + + // -------------------------------------------------------------- /a/b/ + d, err = fs.Open("/a/b/") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "c": + aBC = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "3": + aB3 = true + case "4": + aB4 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB3 && aB4 && aBC) { + t.Fatal(aB4, aB4, aBC) + } + + // --------------------------------------------------------------- /a/b + d, err = fs.Open("/a/b") + if err != nil { + t.Fatal(err) + } + + stat, err = d.Stat() + if err != nil { + t.Fatal(err) + } + + if g, e := stat.IsDir(), true; g != e { + t.Fatal(g, e) + } + + list, err = d.Readdir(0) + if err != nil { + t.Fatal(err) + } + + if g, e := len(list), 3; g != e { + t.Fatal(g, e) + } + + for _, v := range list { + switch v.IsDir() { + case true: + switch v.Name() { + case "c": + aBC = true + default: + t.Fatal(v.Name()) + } + default: + switch v.Name() { + case "3": + aB3 = true + case "4": + aB4 = true + default: + t.Fatal(v.Name()) + } + } + } + if !(aB3 && aB4 && aBC) { + t.Fatal(aB4, aB4, aBC) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/introspection.go b/Godeps/_workspace/src/github.com/cznic/ql/introspection.go new file mode 100644 index 00000000000..61ac9a928d2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/introspection.go @@ -0,0 +1,625 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "fmt" + "go/ast" + "reflect" + "strings" + "sync" +) + +var ( + schemaCache = map[reflect.Type]*StructInfo{} + schemaMu sync.RWMutex +) + +// StructInfo describes a struct type. An instance of StructInfo obtained from +// StructSchema is shared and must not be mutated. That includes the values +// pointed to by the elements of Fields and Indices. +type StructInfo struct { + Fields []*StructField // Fields describe the considered fields of a struct type. + HasID bool // Whether the struct has a considered field named ID of type int64. + Indices []*StructIndex // Indices describe indices defined by the index or uindex ql tags. + IsPtr bool // Whether the StructInfo was derived from a pointer to a struct. +} + +// StructIndex describes an index defined by the ql tag index or uindex. +type StructIndex struct { + ColumnName string // Name of the column the index is on. + Name string // Name of the index. + Unique bool // Whether the index is unique. +} + +// StructField describes a considered field of a struct type. +type StructField struct { + Index int // Index is the index of the field for reflect.Value.Field. + IsID bool // Whether the field corresponds to record id(). + IsPtr bool // Whether the field is a pointer type. + MarshalType reflect.Type // The reflect.Type a field must be converted to when marshaling or nil when it is assignable directly. (Field->value) + Name string // Field name or value of the name tag (like in `ql:"name foo"`). + ReflectType reflect.Type // The reflect.Type of the field. + Tags map[string]string // QL tags of this field. (`ql:"a, b c, d"` -> {"a": "", "b": "c", "d": ""}) + Type Type // QL type of the field. + UnmarshalType reflect.Type // The reflect.Type a value must be converted to when unmarshaling or nil when it is assignable directly. (Field<-value) + ZeroPtr reflect.Value // The reflect.Zero value of the field if it's a pointer type. +} + +func (s *StructField) check(v interface{}) error { + t := reflect.TypeOf(v) + if !s.ReflectType.AssignableTo(t) { + if !s.ReflectType.ConvertibleTo(t) { + return fmt.Errorf("type %s (%v) cannot be converted to %T", s.ReflectType.Name(), s.ReflectType.Kind(), t.Name()) + } + + s.MarshalType = t + } + + if !t.AssignableTo(s.ReflectType) { + if !t.ConvertibleTo(s.ReflectType) { + return fmt.Errorf("type %s (%v) cannot be converted to %T", t.Name(), t.Kind(), s.ReflectType.Name()) + } + + s.UnmarshalType = s.ReflectType + } + return nil +} + +func parseTag(s string) map[string]string { + m := map[string]string{} + for _, v := range strings.Split(s, ",") { + v = strings.TrimSpace(v) + switch n := strings.IndexRune(v, ' '); { + case n < 0: + m[v] = "" + default: + m[v[:n]] = v[n+1:] + } + } + return m +} + +// StructSchema returns StructInfo for v which must be a struct instance or a +// pointer to a struct. The info is computed only once for every type. +// Subsequent calls to StructSchema for the same type return a cached +// StructInfo. +// +// Note: The returned StructSchema is shared and must be not mutated, including +// any other data structures it may point to. +func StructSchema(v interface{}) (*StructInfo, error) { + if v == nil { + return nil, fmt.Errorf("cannot derive schema for %T(%v)", v, v) + } + + typ := reflect.TypeOf(v) + schemaMu.RLock() + if r, ok := schemaCache[typ]; ok { + schemaMu.RUnlock() + return r, nil + } + + schemaMu.RUnlock() + var schemaPtr bool + t := typ + if t.Kind() == reflect.Ptr { + t = t.Elem() + schemaPtr = true + } + if k := t.Kind(); k != reflect.Struct { + return nil, fmt.Errorf("cannot derive schema for type %T (%v)", v, k) + } + + r := &StructInfo{IsPtr: schemaPtr} + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + fn := f.Name + if !ast.IsExported(fn) { + continue + } + + tags := parseTag(f.Tag.Get("ql")) + if _, ok := tags["-"]; ok { + continue + } + + if s := tags["name"]; s != "" { + fn = s + } + + if fn == "ID" && f.Type.Kind() == reflect.Int64 { + r.HasID = true + } + var ix, unique bool + var xn string + xfn := fn + if s := tags["index"]; s != "" { + if _, ok := tags["uindex"]; ok { + return nil, fmt.Errorf("both index and uindex in QL struct tag") + } + + ix, xn = true, s + } else if s := tags["uindex"]; s != "" { + if _, ok := tags["index"]; ok { + return nil, fmt.Errorf("both index and uindex in QL struct tag") + } + + ix, unique, xn = true, true, s + } + if ix { + if fn == "ID" && r.HasID { + xfn = "id()" + } + r.Indices = append(r.Indices, &StructIndex{Name: xn, ColumnName: xfn, Unique: unique}) + } + + sf := &StructField{Index: i, Name: fn, Tags: tags, Type: Type(-1), ReflectType: f.Type} + fk := sf.ReflectType.Kind() + if fk == reflect.Ptr { + sf.IsPtr = true + sf.ZeroPtr = reflect.Zero(sf.ReflectType) + sf.ReflectType = sf.ReflectType.Elem() + fk = sf.ReflectType.Kind() + } + + switch fk { + case reflect.Bool: + sf.Type = Bool + if err := sf.check(false); err != nil { + return nil, err + } + case reflect.Int, reflect.Uint: + return nil, fmt.Errorf("only integers of fixed size can be used to derive a schema: %v", fk) + case reflect.Int8: + sf.Type = Int8 + if err := sf.check(int8(0)); err != nil { + return nil, err + } + case reflect.Int16: + if err := sf.check(int16(0)); err != nil { + return nil, err + } + sf.Type = Int16 + case reflect.Int32: + if err := sf.check(int32(0)); err != nil { + return nil, err + } + sf.Type = Int32 + case reflect.Int64: + if sf.ReflectType.Name() == "Duration" && sf.ReflectType.PkgPath() == "time" { + sf.Type = Duration + break + } + + sf.Type = Int64 + if err := sf.check(int64(0)); err != nil { + return nil, err + } + case reflect.Uint8: + sf.Type = Uint8 + if err := sf.check(uint8(0)); err != nil { + return nil, err + } + case reflect.Uint16: + sf.Type = Uint16 + if err := sf.check(uint16(0)); err != nil { + return nil, err + } + case reflect.Uint32: + sf.Type = Uint32 + if err := sf.check(uint32(0)); err != nil { + return nil, err + } + case reflect.Uint64: + sf.Type = Uint64 + if err := sf.check(uint64(0)); err != nil { + return nil, err + } + case reflect.Float32: + sf.Type = Float32 + if err := sf.check(float32(0)); err != nil { + return nil, err + } + case reflect.Float64: + sf.Type = Float64 + if err := sf.check(float64(0)); err != nil { + return nil, err + } + case reflect.Complex64: + sf.Type = Complex64 + if err := sf.check(complex64(0)); err != nil { + return nil, err + } + case reflect.Complex128: + sf.Type = Complex128 + if err := sf.check(complex128(0)); err != nil { + return nil, err + } + case reflect.Slice: + sf.Type = Blob + if err := sf.check([]byte(nil)); err != nil { + return nil, err + } + case reflect.Struct: + switch sf.ReflectType.PkgPath() { + case "math/big": + switch sf.ReflectType.Name() { + case "Int": + sf.Type = BigInt + case "Rat": + sf.Type = BigRat + } + case "time": + switch sf.ReflectType.Name() { + case "Time": + sf.Type = Time + } + } + case reflect.String: + sf.Type = String + if err := sf.check(""); err != nil { + return nil, err + } + } + + if sf.Type < 0 { + return nil, fmt.Errorf("cannot derive schema for type %s (%v)", sf.ReflectType.Name(), fk) + } + + sf.IsID = fn == "ID" && r.HasID + r.Fields = append(r.Fields, sf) + } + + schemaMu.Lock() + schemaCache[typ] = r + if t != typ { + r2 := *r + r2.IsPtr = false + schemaCache[t] = &r2 + } + schemaMu.Unlock() + return r, nil +} + +// MustStructSchema is like StructSchema but panics on error. It simplifies +// safe initialization of global variables holding StructInfo. +// +// MustStructSchema is safe for concurrent use by multiple goroutines. +func MustStructSchema(v interface{}) *StructInfo { + s, err := StructSchema(v) + if err != nil { + panic(err) + } + + return s +} + +// SchemaOptions amend the result of Schema. +type SchemaOptions struct { + // Don't wrap the CREATE statement(s) in a transaction. + NoTransaction bool + + // Don't insert the IF NOT EXISTS clause in the CREATE statement(s). + NoIfNotExists bool + + // Do not strip the "pkg." part from type name "pkg.Type", produce + // "pkg_Type" table name instead. Applies only when no name is passed + // to Schema(). + KeepPrefix bool +} + +var zeroSchemaOptions SchemaOptions + +// Schema returns a CREATE TABLE/INDEX statement(s) for a table derived from a +// struct or an error, if any. The table is named using the name parameter. If +// name is an empty string then the type name of the struct is used while non +// conforming characters are replaced by underscores. Value v can be also a +// pointer to a struct. +// +// Every considered struct field type must be one of the QL types or a type +// convertible to string, bool, int*, uint*, float* or complex* type or pointer +// to such type. Integers with a width dependent on the architecture can not be +// used. Only exported fields are considered. If an exported field QL tag +// contains "-" (`ql:"-"`) then such field is not considered. A field with name +// ID, having type int64, corresponds to id() - and is thus not a part of the +// CREATE statement. A field QL tag containing "index name" or "uindex name" +// triggers additionally creating an index or unique index on the respective +// field. Fields can be renamed using a QL tag "name newName". Fields are +// considered in the order of appearance. A QL tag is a struct tag part +// prefixed by "ql:". Tags can be combined, for example: +// +// type T struct { +// Foo string `ql:"index xFoo, name Bar"` +// } +// +// If opts.NoTransaction == true then the statement(s) are not wrapped in a +// transaction. If opt.NoIfNotExists == true then the CREATE statement(s) omits +// the IF NOT EXISTS clause. Passing nil opts is equal to passing +// &SchemaOptions{} +// +// Schema is safe for concurrent use by multiple goroutines. +func Schema(v interface{}, name string, opt *SchemaOptions) (List, error) { + if opt == nil { + opt = &zeroSchemaOptions + } + s, err := StructSchema(v) + if err != nil { + return List{}, err + } + + var buf bytes.Buffer + if !opt.NoTransaction { + buf.WriteString("BEGIN TRANSACTION; ") + } + buf.WriteString("CREATE TABLE ") + if !opt.NoIfNotExists { + buf.WriteString("IF NOT EXISTS ") + } + if name == "" { + name = fmt.Sprintf("%T", v) + if !opt.KeepPrefix { + a := strings.Split(name, ".") + if l := len(a); l > 1 { + name = a[l-1] + } + } + nm := []rune{} + for _, v := range name { + switch { + case v >= '0' && v <= '9' || v == '_' || v >= 'a' && v <= 'z' || v >= 'A' && v <= 'Z': + // ok + default: + v = '_' + } + nm = append(nm, v) + } + name = string(nm) + } + buf.WriteString(name + " (") + for _, v := range s.Fields { + if v.IsID { + continue + } + + buf.WriteString(fmt.Sprintf("%s %s, ", v.Name, v.Type)) + } + buf.WriteString("); ") + for _, v := range s.Indices { + buf.WriteString("CREATE ") + if v.Unique { + buf.WriteString("UNIQUE ") + } + buf.WriteString("INDEX ") + if !opt.NoIfNotExists { + buf.WriteString("IF NOT EXISTS ") + } + buf.WriteString(fmt.Sprintf("%s ON %s (%s); ", v.Name, name, v.ColumnName)) + } + if !opt.NoTransaction { + buf.WriteString("COMMIT; ") + } + l, err := Compile(buf.String()) + if err != nil { + return List{}, fmt.Errorf("%s: %v", buf.String(), err) + } + + return l, nil +} + +// MustSchema is like Schema but panics on error. It simplifies safe +// initialization of global variables holding compiled schemas. +// +// MustSchema is safe for concurrent use by multiple goroutines. +func MustSchema(v interface{}, name string, opt *SchemaOptions) List { + l, err := Schema(v, name, opt) + if err != nil { + panic(err) + } + + return l +} + +// Marshal converts, in the order of appearance, fields of a struct instance v +// to []interface{} or an error, if any. Value v can be also a pointer to a +// struct. +// +// Every considered struct field type must be one of the QL types or a type +// convertible to string, bool, int*, uint*, float* or complex* type or pointer +// to such type. Integers with a width dependent on the architecture can not be +// used. Only exported fields are considered. If an exported field QL tag +// contains "-" then such field is not considered. A QL tag is a struct tag +// part prefixed by "ql:". Field with name ID, having type int64, corresponds +// to id() - and is thus not part of the result. +// +// Marshal is safe for concurrent use by multiple goroutines. +func Marshal(v interface{}) ([]interface{}, error) { + s, err := StructSchema(v) + if err != nil { + return nil, err + } + + val := reflect.ValueOf(v) + if s.IsPtr { + val = val.Elem() + } + n := len(s.Fields) + if s.HasID { + n-- + } + r := make([]interface{}, n) + j := 0 + for _, v := range s.Fields { + if v.IsID { + continue + } + + f := val.Field(v.Index) + if v.IsPtr { + if f.IsNil() { + r[j] = nil + j++ + continue + } + + f = f.Elem() + } + if m := v.MarshalType; m != nil { + f = f.Convert(m) + } + r[j] = f.Interface() + j++ + } + return r, nil +} + +// MustMarshal is like Marshal but panics on error. It simplifies marshaling of +// "safe" types, like eg. those which were already verified by Schema or +// MustSchema. When the underlying Marshal returns an error, MustMarshal +// panics. +// +// MustMarshal is safe for concurrent use by multiple goroutines. +func MustMarshal(v interface{}) []interface{} { + r, err := Marshal(v) + if err != nil { + panic(err) + } + + return r +} + +// Unmarshal stores data from []interface{} in the struct value pointed to by +// v. +// +// Every considered struct field type must be one of the QL types or a type +// convertible to string, bool, int*, uint*, float* or complex* type or pointer +// to such type. Integers with a width dependent on the architecture can not be +// used. Only exported fields are considered. If an exported field QL tag +// contains "-" then such field is not considered. A QL tag is a struct tag +// part prefixed by "ql:". Fields are considered in the order of appearance. +// Types of values in data must be compatible with the corresponding considered +// field of v. +// +// If the struct has no ID field then the number of values in data must be equal +// to the number of considered fields of v. +// +// type T struct { +// A bool +// B string +// } +// +// Assuming the schema is +// +// CREATE TABLE T (A bool, B string); +// +// Data might be a result of queries like +// +// SELECT * FROM T; +// SELECT A, B FROM T; +// +// If the struct has a considered ID field then the number of values in data +// must be equal to the number of considered fields in v - or one less. In the +// later case the ID field is not set. +// +// type U struct { +// ID int64 +// A bool +// B string +// } +// +// Assuming the schema is +// +// CREATE TABLE T (A bool, B string); +// +// Data might be a result of queries like +// +// SELECT * FROM T; // ID not set +// SELECT A, B FROM T; // ID not set +// SELECT id(), A, B FROM T; // ID is set +// +// To unmarshal a value from data into a pointer field of v, Unmarshal first +// handles the case of the value being nil. In that case, Unmarshal sets the +// pointer to nil. Otherwise, Unmarshal unmarshals the data value into value +// pointed at by the pointer. If the pointer is nil, Unmarshal allocates a new +// value for it to point to. +// +// Unmarshal is safe for concurrent use by multiple goroutines. +func Unmarshal(v interface{}, data []interface{}) (err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + if err, ok = r.(error); !ok { + err = fmt.Errorf("%v", r) + } + err = fmt.Errorf("unmarshal: %v", err) + } + }() + + s, err := StructSchema(v) + if err != nil { + return err + } + + if !s.IsPtr { + return fmt.Errorf("unmarshal: need a pointer to a struct") + } + + id := false + nv, nf := len(data), len(s.Fields) + switch s.HasID { + case true: + switch { + case nv == nf: + id = true + case nv == nf-1: + // ok + default: + return fmt.Errorf("unmarshal: got %d values, need %d or %d", nv, nf-1, nf) + } + default: + switch { + case nv == nf: + // ok + default: + return fmt.Errorf("unmarshal: got %d values, need %d", nv, nf) + } + } + + j := 0 + vVal := reflect.ValueOf(v) + if s.IsPtr { + vVal = vVal.Elem() + } + for _, sf := range s.Fields { + if sf.IsID && !id { + continue + } + + d := data[j] + val := reflect.ValueOf(d) + j++ + + fVal := vVal.Field(sf.Index) + if u := sf.UnmarshalType; u != nil { + val = val.Convert(u) + } + if !sf.IsPtr { + fVal.Set(val) + continue + } + + if d == nil { + fVal.Set(sf.ZeroPtr) + continue + } + + if fVal.IsNil() { + fVal.Set(reflect.New(sf.ReflectType)) + } + + fVal.Elem().Set(val) + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/introspection_test.go b/Godeps/_workspace/src/github.com/cznic/ql/introspection_test.go new file mode 100644 index 00000000000..2e12b148853 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/introspection_test.go @@ -0,0 +1,1073 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "fmt" + "math/big" + //"reflect" + "testing" + "time" + + "github.com/cznic/mathutil" +) + +type ( + testSchema struct { + a int8 + ID int64 + A int8 + b int + B int `ql:"-"` + } + + testSchema2 struct{} + + testSchema3 struct { + a int8 + ID uint64 + A int8 + b int + B int `ql:"-"` + c bool + C bool `ql:"name cc"` + } + + testSchema4 struct { + a int8 + ID int64 `ql:"name id"` + A int8 + b int + B int `ql:"-"` + c bool + C bool `ql:"name cc"` + } + + testSchema5 struct { + I int `ql:"index x,uindex u"` + } + + testSchema6 struct { + A string `ql:"index x"` + } + + testSchema7 struct { + A int64 + B string `ql:"uindex x"` + C bool + } + + testSchema8 struct { + A bool + //B int + C int8 + D int16 + E int32 + F int64 + //G uint + H uint8 + I uint16 + J uint32 + K uint64 + L float32 + M float64 + N complex64 + O complex128 + P []byte + Q big.Int + R big.Rat + S string + T time.Time + U time.Duration + PA *bool + //PB *int + PC *int8 + PD *int16 + PE *int32 + PF *int64 + //PG *uint + PH *uint8 + PI *uint16 + PJ *uint32 + PK *uint64 + PL *float32 + PM *float64 + PN *complex64 + PO *complex128 + PP *[]byte + PQ *big.Int + PR *big.Rat + PS *string + PT *time.Time + PU *time.Duration + } + + testSchema9 struct { + i int + ID int64 `ql:"index xID"` + Other string `ql:"-"` + DepartmentName string `ql:"uindex xDepartmentName"` + } +) + +const ( + testSchemaSFFF = "begin transaction; create table if not exists testSchema (A int8); commit;" + testSchemaSFFT = "begin transaction; create table if not exists ql_testSchema (A int8); commit;" + testSchemaSFTF = "begin transaction; create table testSchema (A int8); commit;" + testSchemaSFTT = "begin transaction; create table ql_testSchema (A int8); commit;" + testSchemaSTFF = "create table if not exists testSchema (A int8)" + testSchemaSTFT = "create table if not exists ql_testSchema (A int8)" + testSchemaSTTF = "create table testSchema (A int8)" + testSchemaSTTT = "create table ql_testSchema (A int8)" + testSchema3S = "begin transaction; create table if not exists testSchema3 (ID uint64, A int8, cc bool); commit;" + testSchema4S = "begin transaction; create table if not exists testSchema4 (id int64, A int8, cc bool); commit;" + testSchema6S = "create table testSchema6 (A string); create index x on testSchema6 (A);" + testSchema7S = "begin transaction; create table testSchema7 (A int64, B string, C bool); create unique index x on testSchema7 (B); commit;" + testSchema8S = ` + begin transaction; + create table if not exists testSchema8 ( + A bool, + //B int64, + C int8, + D int16, + E int32, + F int64, + //G uint64, + H uint8, + I uint16, + J uint32, + K uint64, + L float32, + M float64, + N complex64, + O complex128, + P blob, + Q bigInt, + R bigRat, + S string, + T time, + U duration, + PA bool, + //PB int64, + PC int8, + PD int16, + PE int32, + PF int64, + //PG uint64, + PH uint8, + PI uint16, + PJ uint32, + PK uint64, + PL float32, + PM float64, + PN complex64, + PO complex128, + PP blob, + PQ bigInt, + PR bigRat, + PS string, + PT time, + PU duration, + ); + commit;` + testSchema9S = ` + begin transaction; + create table if not exists testSchema9 (DepartmentName string); + create index if not exists xID on testSchema9 (id()); + create unique index if not exists xDepartmentName on testSchema9 (DepartmentName); + commit;` +) + +func TestSchema(t *testing.T) { + tab := []struct { + inst interface{} + name string + opts *SchemaOptions + err bool + s string + }{ + // 0 + {inst: nil, err: true}, + {inst: interface{}(nil), err: true}, + {testSchema{}, "", nil, false, testSchemaSFFF}, + {testSchema{}, "", &SchemaOptions{}, false, testSchemaSFFF}, + {testSchema{}, "", &SchemaOptions{KeepPrefix: true}, false, testSchemaSFFT}, + // 5 + {testSchema{}, "", &SchemaOptions{NoIfNotExists: true}, false, testSchemaSFTF}, + {testSchema{}, "", &SchemaOptions{NoIfNotExists: true, KeepPrefix: true}, false, testSchemaSFTT}, + {testSchema{}, "", &SchemaOptions{NoTransaction: true}, false, testSchemaSTFF}, + {testSchema{}, "", &SchemaOptions{NoTransaction: true, KeepPrefix: true}, false, testSchemaSTFT}, + {testSchema{}, "", &SchemaOptions{NoTransaction: true, NoIfNotExists: true}, false, testSchemaSTTF}, + // 10 + {testSchema{}, "", &SchemaOptions{NoTransaction: true, NoIfNotExists: true, KeepPrefix: true}, false, testSchemaSTTT}, + {testSchema2{}, "", nil, true, ""}, + {testSchema3{}, "", nil, false, testSchema3S}, + {testSchema4{}, "", nil, false, testSchema4S}, + {testSchema5{}, "", nil, true, ""}, + // 15 + {testSchema6{}, "", &SchemaOptions{NoTransaction: true, NoIfNotExists: true}, false, testSchema6S}, + {testSchema7{}, "", &SchemaOptions{NoIfNotExists: true}, false, testSchema7S}, + {testSchema8{}, "", nil, false, testSchema8S}, + {&testSchema8{}, "", nil, false, testSchema8S}, + {&testSchema9{}, "", nil, false, testSchema9S}, + } + + for iTest, test := range tab { + l, err := Schema(test.inst, test.name, test.opts) + if g, e := err != nil, test.err; g != e { + t.Fatal(iTest, g, e, err) + } + + if err != nil { + t.Log(iTest, err) + continue + } + + s, err := Compile(test.s) + if err != nil { + panic("internal error 070") + } + + if g, e := l.String(), s.String(); g != e { + t.Fatalf("%d\n----\n%s\n----\n%s", iTest, g, e) + } + } +} + +func ExampleSchema() { + type department struct { + a int // unexported -> ignored + ID int64 `ql:"index xID"` + Other string `xml:"-" ql:"-"` // ignored by QL tag + DepartmentName string `ql:"name Name, uindex xName" json:"foo"` + m bool + HQ int32 + z string + } + + schema := MustSchema((*department)(nil), "", nil) + sel := MustCompile(` + SELECT * FROM __Table; + SELECT * FROM __Column; + SELECT * FROM __Index;`, + ) + fmt.Print(schema) + + db, err := OpenMem() + if err != nil { + panic(err) + } + + if _, _, err = db.Execute(NewRWCtx(), schema); err != nil { + panic(err) + } + + rs, _, err := db.Execute(nil, sel) + if err != nil { + panic(err) + } + + for _, rs := range rs { + fmt.Println("----") + if err = rs.Do(true, func(data []interface{}) (bool, error) { + fmt.Println(data) + return true, nil + }); err != nil { + panic(err) + } + } + // Output: + // BEGIN TRANSACTION; + // CREATE TABLE IF NOT EXISTS department (Name string, HQ int32); + // CREATE INDEX IF NOT EXISTS xID ON department (id()); + // CREATE UNIQUE INDEX IF NOT EXISTS xName ON department (Name); + // COMMIT; + // ---- + // [Name Schema] + // [department CREATE TABLE department (Name string, HQ int32);] + // ---- + // [TableName Ordinal Name Type] + // [department 1 Name string] + // [department 2 HQ int32] + // ---- + // [TableName ColumnName Name IsUnique] + // [department id() xID false] + // [department Name xName true] +} + +func TestMarshal(t *testing.T) { + now := time.Now() + dur := time.Millisecond + schema8 := testSchema8{ + A: true, + //B: 1, + C: 2, + D: 3, + E: 4, + F: 5, + //G: 6, + H: 7, + I: 8, + J: 9, + K: 10, + L: 11, + M: 12, + N: -1, + O: -2, + P: []byte("abc"), + Q: *big.NewInt(1), + R: *big.NewRat(3, 2), + S: "string", + T: now, + U: dur, + } + schema8.PA = &schema8.A + //schema8.PB = &schema8.B + schema8.PC = &schema8.C + schema8.PD = &schema8.D + schema8.PE = &schema8.E + schema8.PF = &schema8.F + //schema8.PG = &schema8.G + schema8.PH = &schema8.H + schema8.PI = &schema8.I + schema8.PJ = &schema8.J + schema8.PK = &schema8.K + schema8.PL = &schema8.L + schema8.PM = &schema8.M + schema8.PN = &schema8.N + schema8.PO = &schema8.O + schema8.PP = &schema8.P + schema8.PQ = &schema8.Q + schema8.PR = &schema8.R + schema8.PS = &schema8.S + schema8.PT = &schema8.T + schema8.PU = &schema8.U + + type u int + tab := []struct { + inst interface{} + err bool + r []interface{} + }{ + {42, true, nil}, + {new(u), true, nil}, + {testSchema8{}, false, []interface{}{ + false, + //int64(0), + int8(0), + int16(0), + int32(0), + int64(0), + //uint64(0), + uint8(0), + uint16(0), + uint32(0), + uint64(0), + float32(0), + float64(0), + complex64(0), + complex128(0), + []byte(nil), + big.Int{}, + big.Rat{}, + "", + time.Time{}, + time.Duration(0), + nil, + //nil, + nil, + nil, + nil, + nil, + //nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }}, + {&testSchema8{}, false, []interface{}{ + false, + //int64(0), + int8(0), + int16(0), + int32(0), + int64(0), + //uint64(0), + uint8(0), + uint16(0), + uint32(0), + uint64(0), + float32(0), + float64(0), + complex64(0), + complex128(0), + []byte(nil), + big.Int{}, + big.Rat{}, + "", + time.Time{}, + time.Duration(0), + nil, + //nil, + nil, + nil, + nil, + nil, + //nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }}, + {schema8, false, []interface{}{ + true, + //int64(1), + int8(2), + int16(3), + int32(4), + int64(5), + //uint64(6), + uint8(7), + uint16(8), + uint32(9), + uint64(10), + float32(11), + float64(12), + complex64(-1), + complex128(-2), + []byte("abc"), + *big.NewInt(1), + *big.NewRat(3, 2), + "string", + now, + dur, + true, + //int64(1), + int8(2), + int16(3), + int32(4), + int64(5), + //uint64(6), + uint8(7), + uint16(8), + uint32(9), + uint64(10), + float32(11), + float64(12), + complex64(-1), + complex128(-2), + []byte("abc"), + *big.NewInt(1), + *big.NewRat(3, 2), + "string", + now, + dur, + }}, + {&schema8, false, []interface{}{ + true, + //int64(1), + int8(2), + int16(3), + int32(4), + int64(5), + //uint64(6), + uint8(7), + uint16(8), + uint32(9), + uint64(10), + float32(11), + float64(12), + complex64(-1), + complex128(-2), + []byte("abc"), + *big.NewInt(1), + *big.NewRat(3, 2), + "string", + now, + dur, + true, + //int64(1), + int8(2), + int16(3), + int32(4), + int64(5), + //uint64(6), + uint8(7), + uint16(8), + uint32(9), + uint64(10), + float32(11), + float64(12), + complex64(-1), + complex128(-2), + []byte("abc"), + *big.NewInt(1), + *big.NewRat(3, 2), + "string", + now, + dur, + }}, + } + for iTest, test := range tab { + r, err := Marshal(test.inst) + if g, e := err != nil, test.err; g != e { + t.Fatal(iTest, g, e) + } + + if err != nil { + t.Log(err) + continue + } + + for i := 0; i < mathutil.Min(len(r), len(test.r)); i++ { + g, e := r[i], test.r[i] + use(e) + switch x := g.(type) { + case bool: + switch y := e.(type) { + case bool: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case int: + switch y := e.(type) { + case int64: + if int64(x) != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case int8: + switch y := e.(type) { + case int8: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case int16: + switch y := e.(type) { + case int16: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case int32: + switch y := e.(type) { + case int32: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case int64: + switch y := e.(type) { + case int64: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case uint: + switch y := e.(type) { + case uint64: + if uint64(x) != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case uint8: + switch y := e.(type) { + case uint8: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case uint16: + switch y := e.(type) { + case uint16: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case uint32: + switch y := e.(type) { + case uint32: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case uint64: + switch y := e.(type) { + case uint64: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case float32: + switch y := e.(type) { + case float32: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case float64: + switch y := e.(type) { + case float64: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case complex64: + switch y := e.(type) { + case complex64: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case complex128: + switch y := e.(type) { + case complex128: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case []byte: + switch y := e.(type) { + case []byte: + if bytes.Compare(x, y) != 0 { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case big.Int: + switch y := e.(type) { + case big.Int: + if x.Cmp(&y) != 0 { + t.Fatal(iTest, &x, &y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case big.Rat: + switch y := e.(type) { + case big.Rat: + if x.Cmp(&y) != 0 { + t.Fatal(iTest, &x, &y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case string: + switch y := e.(type) { + case string: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case time.Time: + switch y := e.(type) { + case time.Time: + if !x.Equal(y) { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case time.Duration: + switch y := e.(type) { + case time.Duration: + if x != y { + t.Fatal(iTest, x, y) + } + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + case nil: + switch y := e.(type) { + case nil: + // ok + default: + t.Fatalf("%d: %T <-> %T", iTest, x, y) + } + default: + panic(fmt.Errorf("%T", x)) + } + } + + if g, e := len(r), len(test.r); g != e { + t.Fatal(iTest, g, e) + } + + } +} + +func ExampleMarshal() { + type myInt int16 + + type myString string + + type item struct { + ID int64 + Name myString + Qty *myInt // pointer enables nil values + Bar int8 + } + + schema := MustSchema((*item)(nil), "", nil) + ins := MustCompile(` + BEGIN TRANSACTION; + INSERT INTO item VALUES($1, $2, $3); + COMMIT;`, + ) + + db, err := OpenMem() + if err != nil { + panic(err) + } + + ctx := NewRWCtx() + if _, _, err := db.Execute(ctx, schema); err != nil { + panic(err) + } + + if _, _, err := db.Execute(ctx, ins, MustMarshal(&item{Name: "foo", Bar: -1})...); err != nil { + panic(err) + } + + q := myInt(42) + if _, _, err := db.Execute(ctx, ins, MustMarshal(&item{Name: "bar", Qty: &q})...); err != nil { + panic(err) + } + + rs, _, err := db.Run(nil, "SELECT * FROM item ORDER BY id();") + if err != nil { + panic(err) + } + + if err = rs[0].Do(true, func(data []interface{}) (bool, error) { + fmt.Println(data) + return true, nil + }); err != nil { + panic(err) + } + // Output: + // [Name Qty Bar] + // [foo -1] + // [bar 42 0] +} + +func TestUnmarshal0(t *testing.T) { + type t1 struct { + I, J int64 + } + + // ---- value field + v1 := &t1{-1, -2} + if err := Unmarshal(v1, []interface{}{int64(42), int64(314)}); err != nil { + t.Fatal(err) + } + + if g, e := v1.I, int64(42); g != e { + t.Fatal(g, e) + } + + if g, e := v1.J, int64(314); g != e { + t.Fatal(g, e) + } + + type t2 struct { + P *int64 + } + + // ---- nil into nil ptr field + v2 := &t2{P: nil} + if err := Unmarshal(v2, []interface{}{nil}); err != nil { + t.Fatal(err) + } + + if g, e := v2.P, (*int64)(nil); g != e { + t.Fatal(g, e) + } + + v2 = &t2{P: nil} + if err := Unmarshal(v2, []interface{}{interface{}(nil)}); err != nil { + t.Fatal(err) + } + + if g, e := v2.P, (*int64)(nil); g != e { + t.Fatal(g, e) + } + + // ---- nil into non nil ptr field + i := int64(42) + v2 = &t2{P: &i} + if err := Unmarshal(v2, []interface{}{nil}); err != nil { + t.Fatal(err) + } + + if g, e := v2.P, (*int64)(nil); g != e { + t.Fatal(g, e) + } + + if g, e := i, int64(42); g != e { + t.Fatal(g, e) + } + + v2 = &t2{P: &i} + if err := Unmarshal(v2, []interface{}{interface{}(nil)}); err != nil { + t.Fatal(err) + } + + if g, e := v2.P, (*int64)(nil); g != e { + t.Fatal(g, e) + } + + if g, e := i, int64(42); g != e { + t.Fatal(g, e) + } + + // ---- non nil value into non nil ptr field + i = 42 + v2 = &t2{P: &i} + if err := Unmarshal(v2, []interface{}{int64(314)}); err != nil { + t.Fatal(err) + } + + if g, e := v2.P, &i; g != e { + t.Fatal(g, e) + } + + if g, e := i, int64(314); g != e { + t.Fatal(g, e) + } + + // ---- non nil value into nil ptr field + v2 = &t2{P: nil} + if err := Unmarshal(v2, []interface{}{int64(314)}); err != nil { + t.Fatal(err) + } + + if g, e := v2.P != nil, true; g != e { + t.Fatal(g, e) + } + + if g, e := *v2.P, int64(314); g != e { + t.Fatal(g, e) + } +} + +func TestUnmarshal(t *testing.T) { + type myString string + + type t1 struct { + A bool + B myString + } + + type t2 struct { + A bool + ID int64 + B myString + } + + f := func(v interface{}) int64 { + if x, ok := v.(*t2); ok { + return x.ID + } + + return -1 + } + + tab := []struct { + inst interface{} + data []interface{} + err bool + }{ + // 0 + {t1{}, []interface{}{true, "foo"}, true}, // not a ptr + {&t1{}, []interface{}{true}, true}, // too few values + {&t1{}, []interface{}{"foo"}, true}, // too few values + {&t1{}, []interface{}{true, "foo", 42}, true}, // too many values + {&t1{}, []interface{}{"foo", true, 42}, true}, // too many values + // 5 + {&t1{}, []interface{}{true, "foo"}, false}, + {&t1{}, []interface{}{false, "bar"}, false}, + {&t1{}, []interface{}{"bar", "baz"}, true}, + {&t1{}, []interface{}{true, 42.7}, true}, + {&t2{}, []interface{}{1}, true}, // too few values + // 10 + {&t2{}, []interface{}{1, 2, 3, 4}, true}, // too many values + {&t2{}, []interface{}{false, int64(314), "foo"}, false}, + {&t2{}, []interface{}{true, int64(42), "foo"}, false}, + {&t2{}, []interface{}{false, "foo"}, false}, + // 15 + {&t2{}, []interface{}{true, "foo"}, false}, + } + + for iTest, test := range tab { + inst := test.inst + err := Unmarshal(inst, test.data) + if g, e := err != nil, test.err; g != e { + t.Fatal(iTest, g, e) + } + + if err != nil { + t.Log(iTest, err) + continue + } + + data, err := Marshal(inst) + if err != nil { + t.Fatal(iTest, err) + } + + if g, e := len(data), len(test.data); g > e { + t.Fatal(iTest, g, e) + } + + j := 0 + for _, v := range data { + v2 := test.data[j] + j++ + if _, ok := v2.(int64); ok { + if g, e := f(inst), v2; g != e { + t.Fatal(iTest, g, e) + } + + continue + } + + if g, e := v, v2; g != e { + t.Fatal(iTest, g, e) + } + } + } +} + +func ExampleUnmarshal() { + type myString string + + type row struct { + ID int64 + S myString + P *int64 + } + + schema := MustSchema((*row)(nil), "", nil) + ins := MustCompile(` + BEGIN TRANSACTION; + INSERT INTO row VALUES($1, $2); + COMMIT;`, + ) + sel := MustCompile(` + SELECT id(), S, P FROM row ORDER by id(); + SELECT * FROM row ORDER by id();`, + ) + + db, err := OpenMem() + if err != nil { + panic(err) + } + + ctx := NewRWCtx() + if _, _, err = db.Execute(ctx, schema); err != nil { + panic(err) + } + + r := &row{S: "foo"} + if _, _, err = db.Execute(ctx, ins, MustMarshal(r)...); err != nil { + panic(err) + } + + i42 := int64(42) + r = &row{S: "bar", P: &i42} + if _, _, err = db.Execute(ctx, ins, MustMarshal(r)...); err != nil { + panic(err) + } + + rs, _, err := db.Execute(nil, sel) + if err != nil { + panic(err) + } + + for _, rs := range rs { + fmt.Println("----") + if err := rs.Do(false, func(data []interface{}) (bool, error) { + r := &row{} + if err := Unmarshal(r, data); err != nil { + return false, err + } + + fmt.Printf("ID %d, S %q, P ", r.ID, r.S) + switch r.P == nil { + case true: + fmt.Println("") + default: + fmt.Println(*r.P) + } + return true, nil + }); err != nil { + panic(err) + } + } + // Output: + // ---- + // ID 1, S "foo", P + // ID 2, S "bar", P 42 + // ---- + // ID 0, S "foo", P + // ID 0, S "bar", P 42 +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/mem.go b/Godeps/_workspace/src/github.com/cznic/ql/mem.go new file mode 100644 index 00000000000..9f5cc689beb --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/mem.go @@ -0,0 +1,1270 @@ +// Copyright (c) 2014 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Plain memory storage back end. + +package ql + +import ( + "bytes" + "fmt" + "io" + "log" + "math/big" + "time" +) + +var ( + _ btreeIndex = (*memIndex)(nil) + _ btreeIterator = (*memBTreeIterator)(nil) + _ indexIterator = (*xenumerator2)(nil) + _ storage = (*mem)(nil) + _ temp = (*memTemp)(nil) +) + +type memIndex struct { + m *mem + t *xtree + unique bool +} + +func newMemIndex(m *mem, unique bool) *memIndex { + return &memIndex{t: xtreeNew(), unique: unique, m: m} +} + +func (x *memIndex) Clear() error { + x.m.newUndo(undoClearX, 0, []interface{}{x, x.t}) + x.t = xtreeNew() + return nil +} + +func (x *memIndex) Create(indexedValue interface{}, h int64) error { + t := x.t + switch { + case !x.unique: + k := indexKey{indexedValue, h} + x.m.newUndo(undoCreateX, 0, []interface{}{x, k}) + t.Set(k, 0) + case indexedValue == nil: // unique, NULL + k := indexKey{nil, h} + x.m.newUndo(undoCreateX, 0, []interface{}{x, k}) + t.Set(k, 0) + default: // unique, non NULL + k := indexKey{indexedValue, 0} + if _, ok := t.Get(k); ok { //LATER need .Put + return fmt.Errorf("cannot insert into unique index: duplicate value: %v", indexedValue) + } + + x.m.newUndo(undoCreateX, 0, []interface{}{x, k}) + t.Set(k, int(h)) + } + return nil +} + +func (x *memIndex) Delete(indexedValue interface{}, h int64) error { + t := x.t + var k indexKey + var v interface{} + var ok, okv bool + switch { + case !x.unique: + k = indexKey{indexedValue, h} + v, okv = t.Get(k) + ok = t.delete(k) + case indexedValue == nil: // unique, NULL + k = indexKey{nil, h} + v, okv = t.Get(k) + ok = t.delete(k) + default: // unique, non NULL + k = indexKey{indexedValue, 0} + v, okv = t.Get(k) + ok = t.delete(k) + } + if ok { + if okv { + x.m.newUndo(undoDeleteX, int64(v.(int)), []interface{}{x, k}) + } + return nil + } + + return fmt.Errorf("internal error 047") +} + +func (x *memIndex) Drop() error { + x.m.newUndo(undoDropX, 0, []interface{}{x, *x}) + *x = memIndex{} + return nil +} + +func (x *memIndex) Seek(indexedValue interface{}) (indexIterator, bool, error) { + it, hit := x.t.Seek(indexKey{indexedValue, 0}) + return &xenumerator2{*it, x.unique}, hit, nil +} + +func (x *memIndex) SeekFirst() (iter indexIterator, err error) { + it, err := x.t.SeekFirst() + if err != nil { + return nil, err + } + + return &xenumerator2{*it, x.unique}, nil +} + +func (x *memIndex) SeekLast() (iter indexIterator, err error) { + it, err := x.t.SeekLast() + if err != nil { + return nil, err + } + + return &xenumerator2{*it, x.unique}, nil +} + +type xenumerator2 struct { + it xenumerator + unique bool +} + +func (it *xenumerator2) Next() (interface{}, int64, error) { + k, h, err := it.it.Next() + if err != nil { + return nil, -1, err + } + + switch it.unique { + case true: + if k.value == nil { + return nil, k.h, nil + } + + return k.value, h, nil + default: + return k.value, k.h, nil + } +} + +func (it *xenumerator2) Prev() (interface{}, int64, error) { + k, h, err := it.it.Prev() + if err != nil { + return nil, -1, err + } + + switch it.unique { + case true: + if k.value == nil { + return nil, k.h, nil + } + + return k.value, h, nil + default: + return k.value, k.h, nil + } +} + +type memBTreeIterator enumerator + +func (it *memBTreeIterator) Next() (k, v []interface{}, err error) { + return (*enumerator)(it).Next() +} + +type memTemp struct { + tree *tree + store *mem +} + +func (t *memTemp) BeginTransaction() (err error) { + return nil +} + +func (t *memTemp) Get(k []interface{}) (v []interface{}, err error) { + v, _ = t.tree.Get(k) + return +} + +func (t *memTemp) Create(data ...interface{}) (h int64, err error) { + s := t.store + switch n := len(s.recycler); { + case n != 0: + h = int64(s.recycler[n-1]) + s.recycler = s.recycler[:n-1] + s.data[h] = s.clone(data...) + default: + h = int64(len(s.data)) + s.data = append(s.data, s.clone(data...)) + } + return +} + +func (t *memTemp) Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) { + return t.store.Read(dst, h, cols...) +} + +func (*memTemp) Drop() (err error) { return } + +func (t *memTemp) Set(k, v []interface{}) (err error) { + t.tree.Set(append([]interface{}(nil), k...), t.store.clone(v...)) + return +} + +func (t *memTemp) SeekFirst() (e btreeIterator, err error) { + en, err := t.tree.SeekFirst() + if err != nil { + return + } + + return (*memBTreeIterator)(en), nil +} + +const ( + undoCreateNewHandle = iota + undoCreateRecycledHandle + undoUpdate + undoDelete + undoClearX // {0: *memIndex, 1: *xtree} + undoCreateX // {0: *memIndex, 1: indexKey} + undoDeleteX // {0: *memIndex, 1: indexKey} + undoDropX // {0: *memIndex, 1: memIndex} +) + +type undo struct { + tag int + h int64 + data []interface{} +} + +type undos struct { + list []undo + parent *undos +} + +type mem struct { + data [][]interface{} + id int64 + recycler []int + tnl int + rollback *undos +} + +func newMemStorage() (s *mem, err error) { + s = &mem{data: [][]interface{}{nil}} + if err = s.BeginTransaction(); err != nil { + return nil, err + } + + h, err := s.Create() + if h != 1 { + log.Panic("internal error 048") + } + + if err = s.Commit(); err != nil { + return nil, err + } + + return +} + +func (s *mem) OpenIndex(unique bool, handle int64) (btreeIndex, error) { // Never called on the memory backend. + panic("internal error 049") +} + +func (s *mem) newUndo(tag int, h int64, data []interface{}) { + s.rollback.list = append(s.rollback.list, undo{tag, h, data}) +} + +func (s *mem) Acid() bool { return false } + +func (s *mem) Close() (err error) { + *s = mem{} + return +} + +func (s *mem) CreateIndex(unique bool) ( /* handle */ int64, btreeIndex, error) { + return -1, newMemIndex(s, unique), nil // handle of memIndex should never be used +} + +func (s *mem) Name() string { return fmt.Sprintf("/proc/self/mem/%p", s) } // fake, non existing name + +// OpenMem returns a new, empty DB backed by the process' memory. The back end +// has no limits on field/record/table/DB size other than memory available to +// the process. +func OpenMem() (db *DB, err error) { + s, err := newMemStorage() + if err != nil { + return + } + + if db, err = newDB(s); err != nil { + return nil, err + } + + db.isMem = true + return db, nil +} + +func (s *mem) Verify() (allocs int64, err error) { + for _, v := range s.recycler { + if s.data[v] != nil { + return 0, fmt.Errorf("corrupted: non nil free handle %d", s.data[v]) + } + } + + for _, v := range s.data { + if v != nil { + allocs++ + } + } + + if allocs != int64(len(s.data))-1-int64(len(s.recycler)) { + return 0, fmt.Errorf("corrupted: len(data) %d, len(recycler) %d, allocs %d", len(s.data), len(s.recycler), allocs) + } + + return +} + +func (s *mem) String() string { + var b bytes.Buffer + for i, v := range s.data { + b.WriteString(fmt.Sprintf("s.data[%d] %#v\n", i, v)) + } + for i, v := range s.recycler { + b.WriteString(fmt.Sprintf("s.recycler[%d] %v\n", i, v)) + } + return b.String() +} + +func (s *mem) CreateTemp(asc bool) (_ temp, err error) { + st, err := newMemStorage() + if err != nil { + return + } + + return &memTemp{ + tree: treeNew(collators[asc]), + store: st, + }, nil +} + +func (s *mem) ResetID() (err error) { + s.id = 0 + return +} + +func (s *mem) ID() (id int64, err error) { + s.id++ + return s.id, nil +} + +func (s *mem) clone(data ...interface{}) []interface{} { + r := make([]interface{}, len(data)) + for i, v := range data { + switch x := v.(type) { + case nil: + // nop + case idealComplex: + r[i] = complex128(x) + case idealFloat: + r[i] = float64(x) + case idealInt: + r[i] = int64(x) + case idealRune: + r[i] = int32(x) + case idealUint: + r[i] = uint64(x) + case bool: + r[i] = x + case complex64: + r[i] = x + case complex128: + r[i] = x + case float32: + r[i] = x + case float64: + r[i] = x + case int: + r[i] = int64(x) + case int8: + r[i] = x + case int16: + r[i] = x + case int32: + r[i] = x + case int64: + r[i] = x + case string: + r[i] = x + case uint: + r[i] = uint64(x) + case uint8: + r[i] = x + case uint16: + r[i] = x + case uint32: + r[i] = x + case uint64: + r[i] = x + case []byte: + r[i] = append([]byte(nil), x...) + case *big.Int: + r[i] = big.NewInt(0).Set(x) + case *big.Rat: + r[i] = big.NewRat(1, 2).Set(x) + case time.Time: + t := x + r[i] = t + case time.Duration: + r[i] = x + case map[string]interface{}: // map of ids of a cross join + r[i] = x + default: + log.Panic("internal error 050") + } + } + return r +} + +func (s *mem) Create(data ...interface{}) (h int64, err error) { + switch n := len(s.recycler); { + case n != 0: + h = int64(s.recycler[n-1]) + s.recycler = s.recycler[:n-1] + s.data[h] = s.clone(data...) + r := s.rollback + r.list = append(r.list, undo{ + tag: undoCreateRecycledHandle, + h: h, + }) + default: + h = int64(len(s.data)) + s.data = append(s.data, s.clone(data...)) + r := s.rollback + r.list = append(r.list, undo{ + tag: undoCreateNewHandle, + h: h, + }) + } + return +} + +func (s *mem) Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) { + if i := int(h); i != 0 && i < len(s.data) { + d := s.clone(s.data[h]...) + if cols == nil { + return d, nil + } + + for n, dn := len(cols)+2, len(d); dn < n; dn++ { + d = append(d, nil) + } + return d, nil + } + + return nil, errNoDataForHandle +} + +func (s *mem) UpdateRow(h int64, _ []*col, data ...interface{}) (err error) { + return s.Update(h, data...) +} + +func (s *mem) Update(h int64, data ...interface{}) (err error) { + r := s.rollback + r.list = append(r.list, undo{ + tag: undoUpdate, + h: h, + data: s.data[h], + }) + s.data[h] = s.clone(data...) + return +} + +func (s *mem) Delete(h int64, _ ...*col) (err error) { + r := s.rollback + r.list = append(r.list, undo{ + tag: undoDelete, + h: h, + data: s.data[h], + }) + s.recycler = append(s.recycler, int(h)) + s.data[h] = nil + return +} + +func (s *mem) BeginTransaction() (err error) { + s.rollback = &undos{parent: s.rollback} + s.tnl++ + return nil +} + +func (s *mem) Rollback() (err error) { + if s.tnl == 0 { + return errRollbackNotInTransaction + } + + list := s.rollback.list + for i := len(list) - 1; i >= 0; i-- { + undo := list[i] + switch h, data := int(undo.h), undo.data; undo.tag { + case undoCreateNewHandle: + d := s.data + s.data = d[:len(d)-1] + case undoCreateRecycledHandle: + s.data[h] = nil + r := s.recycler + s.recycler = append(r, h) + case undoUpdate: + s.data[h] = data + case undoDelete: + s.data[h] = data + s.recycler = s.recycler[:len(s.recycler)-1] + case undoClearX: + x, t := data[0].(*memIndex), data[1].(*xtree) + x.t = t + case undoCreateX: + x, k := data[0].(*memIndex), data[1].(indexKey) + x.t.delete(k) + case undoDeleteX: + x, k := data[0].(*memIndex), data[1].(indexKey) + x.t.Set(k, h) + case undoDropX: + x, v := data[0].(*memIndex), data[1].(memIndex) + *x = v + default: + log.Panic("internal error 051") + } + } + + s.tnl-- + s.rollback = s.rollback.parent + return nil +} + +func (s *mem) Commit() (err error) { + if s.tnl == 0 { + return errCommitNotInTransaction + } + + s.tnl-- + s.rollback = s.rollback.parent + return nil +} + +// Transaction index B+Tree +//LATER make it just a wrapper of the implementation in btree.go. + +type ( + xd struct { // data page + c int + xd [2*kd + 1]xde + n *xd + p *xd + } + + xde struct { // xd element + k indexKey + v int + } + + // xenumerator captures the state of enumerating a tree. It is returned + // from the Seek* methods. The enumerator is aware of any mutations + // made to the tree in the process of enumerating it and automatically + // resumes the enumeration at the proper key, if possible. + // + // However, once an xenumerator returns io.EOF to signal "no more + // items", it does no more attempt to "resync" on tree mutation(s). In + // other words, io.EOF from an Enumaretor is "sticky" (idempotent). + xenumerator struct { + err error + hit bool + i int + k indexKey + q *xd + t *xtree + ver int64 + } + + // xtree is a B+tree. + xtree struct { + c int + first *xd + last *xd + r interface{} + ver int64 + } + + xxe struct { // xx element + ch interface{} + sep *xd + } + + xx struct { // index page + c int + xx [2*kx + 2]xxe + } +) + +func (a *indexKey) cmp(b *indexKey) int { + r := collate1(a.value, b.value) + if r != 0 { + return r + } + + return int(a.h) - int(b.h) +} + +var ( // R/O zero values + zxd xd + zxde xde + zxx xx + zxxe xxe +) + +func xclr(q interface{}) { + switch xx := q.(type) { + case *xx: + for i := 0; i <= xx.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn + xclr(xx.xx[i].ch) + } + *xx = zxx // GC + case *xd: + *xx = zxd // GC + } +} + +// -------------------------------------------------------------------------- xx + +func xnewX(ch0 interface{}) *xx { + r := &xx{} + r.xx[0].ch = ch0 + return r +} + +func (q *xx) extract(i int) { + q.c-- + if i < q.c { + copy(q.xx[i:], q.xx[i+1:q.c+1]) + q.xx[q.c].ch = q.xx[q.c+1].ch + q.xx[q.c].sep = nil // GC + q.xx[q.c+1] = zxxe // GC + } +} + +func (q *xx) insert(i int, xd *xd, ch interface{}) *xx { + c := q.c + if i < c { + q.xx[c+1].ch = q.xx[c].ch + copy(q.xx[i+2:], q.xx[i+1:c]) + q.xx[i+1].sep = q.xx[i].sep + } + c++ + q.c = c + q.xx[i].sep = xd + q.xx[i+1].ch = ch + return q +} + +func (q *xx) siblings(i int) (l, r *xd) { + if i >= 0 { + if i > 0 { + l = q.xx[i-1].ch.(*xd) + } + if i < q.c { + r = q.xx[i+1].ch.(*xd) + } + } + return +} + +// -------------------------------------------------------------------------- xd + +func (l *xd) mvL(r *xd, c int) { + copy(l.xd[l.c:], r.xd[:c]) + copy(r.xd[:], r.xd[c:r.c]) + l.c += c + r.c -= c +} + +func (l *xd) mvR(r *xd, c int) { + copy(r.xd[c:], r.xd[:r.c]) + copy(r.xd[:c], l.xd[l.c-c:]) + r.c += c + l.c -= c +} + +// ----------------------------------------------------------------------- xtree + +// xtreeNew returns a newly created, empty xtree. The compare function is used +// for key collation. +func xtreeNew() *xtree { + return &xtree{} +} + +// Clear removes all K/V pairs from the tree. +func (t *xtree) Clear() { + if t.r == nil { + return + } + + xclr(t.r) + t.c, t.first, t.last, t.r = 0, nil, nil, nil + t.ver++ +} + +func (t *xtree) cat(p *xx, q, r *xd, pi int) { + t.ver++ + q.mvL(r, r.c) + if r.n != nil { + r.n.p = q + } else { + t.last = q + } + q.n = r.n + if p.c > 1 { + p.extract(pi) + p.xx[pi].ch = q + } else { + t.r = q + } +} + +func (t *xtree) catX(p, q, r *xx, pi int) { + t.ver++ + q.xx[q.c].sep = p.xx[pi].sep + copy(q.xx[q.c+1:], r.xx[:r.c]) + q.c += r.c + 1 + q.xx[q.c].ch = r.xx[r.c].ch + if p.c > 1 { + p.c-- + pc := p.c + if pi < pc { + p.xx[pi].sep = p.xx[pi+1].sep + copy(p.xx[pi+1:], p.xx[pi+2:pc+1]) + p.xx[pc].ch = p.xx[pc+1].ch + p.xx[pc].sep = nil // GC + p.xx[pc+1].ch = nil // GC + } + return + } + + t.r = q +} + +//Delete removes the k's KV pair, if it exists, in which case Delete returns +//true. +func (t *xtree) delete(k indexKey) (ok bool) { + pi := -1 + var p *xx + q := t.r + if q == nil { + return + } + + for { + var i int + i, ok = t.find(q, k) + if ok { + switch xx := q.(type) { + case *xx: + dp := xx.xx[i].sep + switch { + case dp.c > kd: + t.extract(dp, 0) + default: + if xx.c < kx && q != t.r { + t.underflowX(p, &xx, pi, &i) + } + pi = i + 1 + p = xx + q = xx.xx[pi].ch + ok = false + continue + } + case *xd: + t.extract(xx, i) + if xx.c >= kd { + return + } + + if q != t.r { + t.underflow(p, xx, pi) + } else if t.c == 0 { + t.Clear() + } + } + return + } + + switch xx := q.(type) { + case *xx: + if xx.c < kx && q != t.r { + t.underflowX(p, &xx, pi, &i) + } + pi = i + p = xx + q = xx.xx[i].ch + case *xd: + return + } + } +} + +func (t *xtree) extract(q *xd, i int) { // (r int64) { + t.ver++ + //r = q.xd[i].v // prepared for Extract + q.c-- + if i < q.c { + copy(q.xd[i:], q.xd[i+1:q.c+1]) + } + q.xd[q.c] = zxde // GC + t.c-- + return +} + +func (t *xtree) find(q interface{}, k indexKey) (i int, ok bool) { + var mk indexKey + l := 0 + switch xx := q.(type) { + case *xx: + h := xx.c - 1 + for l <= h { + m := (l + h) >> 1 + mk = xx.xx[m].sep.xd[0].k + switch cmp := k.cmp(&mk); { + case cmp > 0: + l = m + 1 + case cmp == 0: + return m, true + default: + h = m - 1 + } + } + case *xd: + h := xx.c - 1 + for l <= h { + m := (l + h) >> 1 + mk = xx.xd[m].k + switch cmp := k.cmp(&mk); { + case cmp > 0: + l = m + 1 + case cmp == 0: + return m, true + default: + h = m - 1 + } + } + } + return l, false +} + +// First returns the first item of the tree in the key collating order, or +// (nil, nil) if the tree is empty. +func (t *xtree) First() (k indexKey, v int) { + if q := t.first; q != nil { + q := &q.xd[0] + k, v = q.k, q.v + } + return +} + +// Get returns the value associated with k and true if it exists. Otherwise Get +// returns (nil, false). +func (t *xtree) Get(k indexKey) (v int, ok bool) { + q := t.r + if q == nil { + return + } + + for { + var i int + if i, ok = t.find(q, k); ok { + switch xx := q.(type) { + case *xx: + return xx.xx[i].sep.xd[0].v, true + case *xd: + return xx.xd[i].v, true + } + } + switch xx := q.(type) { + case *xx: + q = xx.xx[i].ch + default: + return + } + } +} + +func (t *xtree) insert(q *xd, i int, k indexKey, v int) *xd { + t.ver++ + c := q.c + if i < c { + copy(q.xd[i+1:], q.xd[i:c]) + } + c++ + q.c = c + q.xd[i].k, q.xd[i].v = k, v + t.c++ + return q +} + +// Last returns the last item of the tree in the key collating order, or (nil, +// nil) if the tree is empty. +func (t *xtree) Last() (k indexKey, v int) { + if q := t.last; q != nil { + q := &q.xd[q.c-1] + k, v = q.k, q.v + } + return +} + +// Len returns the number of items in the tree. +func (t *xtree) Len() int { + return t.c +} + +func (t *xtree) overflow(p *xx, q *xd, pi, i int, k indexKey, v int) { + t.ver++ + l, r := p.siblings(pi) + + if l != nil && l.c < 2*kd { + l.mvL(q, 1) + t.insert(q, i-1, k, v) + return + } + + if r != nil && r.c < 2*kd { + if i < 2*kd { + q.mvR(r, 1) + t.insert(q, i, k, v) + } else { + t.insert(r, 0, k, v) + } + return + } + + t.split(p, q, pi, i, k, v) +} + +// Seek returns an xenumerator positioned on a an item such that k >= item's +// key. ok reports if k == item.key The xenumerator's position is possibly +// after the last item in the tree. +func (t *xtree) Seek(k indexKey) (e *xenumerator, ok bool) { + q := t.r + if q == nil { + e = &xenumerator{nil, false, 0, k, nil, t, t.ver} + return + } + + for { + var i int + if i, ok = t.find(q, k); ok { + switch xx := q.(type) { + case *xx: + e = &xenumerator{nil, ok, 0, k, xx.xx[i].sep, t, t.ver} + return + case *xd: + e = &xenumerator{nil, ok, i, k, xx, t, t.ver} + return + } + } + switch xx := q.(type) { + case *xx: + q = xx.xx[i].ch + case *xd: + e = &xenumerator{nil, ok, i, k, xx, t, t.ver} + return + } + } +} + +// SeekFirst returns an enumerator positioned on the first KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returned and e will be nil. +func (t *xtree) SeekFirst() (e *xenumerator, err error) { + q := t.first + if q == nil { + return nil, io.EOF + } + + return &xenumerator{nil, true, 0, q.xd[0].k, q, t, t.ver}, nil +} + +// SeekLast returns an enumerator positioned on the last KV pair in the tree, +// if any. For an empty tree, err == io.EOF is returned and e will be nil. +func (t *xtree) SeekLast() (e *xenumerator, err error) { + q := t.last + if q == nil { + return nil, io.EOF + } + + return &xenumerator{nil, true, q.c - 1, q.xd[q.c-1].k, q, t, t.ver}, nil +} + +// Set sets the value associated with k. +func (t *xtree) Set(k indexKey, v int) { + pi := -1 + var p *xx + q := t.r + if q != nil { + for { + i, ok := t.find(q, k) + if ok { + switch xx := q.(type) { + case *xx: + xx.xx[i].sep.xd[0].v = v + case *xd: + xx.xd[i].v = v + } + return + } + + switch xx := q.(type) { + case *xx: + if xx.c > 2*kx { + t.splitX(p, &xx, pi, &i) + } + pi = i + p = xx + q = xx.xx[i].ch + case *xd: + switch { + case xx.c < 2*kd: + t.insert(xx, i, k, v) + default: + t.overflow(p, xx, pi, i, k, v) + } + return + } + } + } + + z := t.insert(&xd{}, 0, k, v) + t.r, t.first, t.last = z, z, z + return +} + +func (t *xtree) split(p *xx, q *xd, pi, i int, k indexKey, v int) { + t.ver++ + r := &xd{} + if q.n != nil { + r.n = q.n + r.n.p = r + } else { + t.last = r + } + q.n = r + r.p = q + + copy(r.xd[:], q.xd[kd:2*kd]) + for i := range q.xd[kd:] { + q.xd[kd+i] = zxde + } + q.c = kd + r.c = kd + if pi >= 0 { + p.insert(pi, r, r) + } else { + t.r = xnewX(q).insert(0, r, r) + } + if i > kd { + t.insert(r, i-kd, k, v) + return + } + + t.insert(q, i, k, v) +} + +func (t *xtree) splitX(p *xx, pp **xx, pi int, i *int) { + t.ver++ + q := *pp + r := &xx{} + copy(r.xx[:], q.xx[kx+1:]) + q.c = kx + r.c = kx + if pi >= 0 { + p.insert(pi, q.xx[kx].sep, r) + } else { + t.r = xnewX(q).insert(0, q.xx[kx].sep, r) + } + q.xx[kx].sep = nil + for i := range q.xx[kx+1:] { + q.xx[kx+i+1] = zxxe + } + if *i > kx { + *pp = r + *i -= kx + 1 + } +} + +func (t *xtree) underflow(p *xx, q *xd, pi int) { + t.ver++ + l, r := p.siblings(pi) + + if l != nil && l.c+q.c >= 2*kd { + l.mvR(q, 1) + } else if r != nil && q.c+r.c >= 2*kd { + q.mvL(r, 1) + r.xd[r.c] = zxde // GC + } else if l != nil { + t.cat(p, l, q, pi-1) + } else { + t.cat(p, q, r, pi) + } +} + +func (t *xtree) underflowX(p *xx, pp **xx, pi int, i *int) { + t.ver++ + var l, r *xx + q := *pp + + if pi >= 0 { + if pi > 0 { + l = p.xx[pi-1].ch.(*xx) + } + if pi < p.c { + r = p.xx[pi+1].ch.(*xx) + } + } + + if l != nil && l.c > kx { + q.xx[q.c+1].ch = q.xx[q.c].ch + copy(q.xx[1:], q.xx[:q.c]) + q.xx[0].ch = l.xx[l.c].ch + q.xx[0].sep = p.xx[pi-1].sep + q.c++ + *i++ + l.c-- + p.xx[pi-1].sep = l.xx[l.c].sep + return + } + + if r != nil && r.c > kx { + q.xx[q.c].sep = p.xx[pi].sep + q.c++ + q.xx[q.c].ch = r.xx[0].ch + p.xx[pi].sep = r.xx[0].sep + copy(r.xx[:], r.xx[1:r.c]) + r.c-- + rc := r.c + r.xx[rc].ch = r.xx[rc+1].ch + r.xx[rc].sep = nil + r.xx[rc+1].ch = nil + return + } + + if l != nil { + *i += l.c + 1 + t.catX(p, l, q, pi-1) + *pp = l + return + } + + t.catX(p, q, r, pi) +} + +// ----------------------------------------------------------------- xenumerator + +// Next returns the currently enumerated item, if it exists and moves to the +// next item in the key collation order. If there is no item to return, err == +// io.EOF is returned. +func (e *xenumerator) Next() (k indexKey, v int64, err error) { + if err = e.err; err != nil { + return + } + + if e.ver != e.t.ver { + f, hit := e.t.Seek(e.k) + if !e.hit && hit { + if err = f.next(); err != nil { + return + } + } + + *e = *f + } + if e.q == nil { + e.err, err = io.EOF, io.EOF + return + } + + if e.i >= e.q.c { + if err = e.next(); err != nil { + return + } + } + + i := e.q.xd[e.i] + k, v = i.k, int64(i.v) + e.k, e.hit = k, false + e.next() + return +} + +func (e *xenumerator) next() error { + if e.q == nil { + e.err = io.EOF + return io.EOF + } + + switch { + case e.i < e.q.c-1: + e.i++ + default: + if e.q, e.i = e.q.n, 0; e.q == nil { + e.err = io.EOF + } + } + return e.err +} + +// Prev returns the currently enumerated item, if it exists and moves to the +// previous item in the key collation order. If there is no item to return, err +// == io.EOF is returned. +func (e *xenumerator) Prev() (k indexKey, v int64, err error) { + if err = e.err; err != nil { + return + } + + if e.ver != e.t.ver { + f, hit := e.t.Seek(e.k) + if !e.hit && hit { + if err = f.prev(); err != nil { + return + } + } + + *e = *f + } + if e.q == nil { + e.err, err = io.EOF, io.EOF + return + } + + if e.i >= e.q.c { + if err = e.next(); err != nil { + return + } + } + + i := e.q.xd[e.i] + k, v = i.k, int64(i.v) + e.k, e.hit = k, false + e.prev() + return +} + +func (e *xenumerator) prev() error { + if e.q == nil { + e.err = io.EOF + return io.EOF + } + + switch { + case e.i > 0: + e.i-- + default: + if e.q = e.q.p; e.q == nil { + e.err = io.EOF + break + } + + e.i = e.q.c - 1 + } + return e.err +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/parser.go b/Godeps/_workspace/src/github.com/cznic/ql/parser.go new file mode 100644 index 00000000000..a357e61a242 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/parser.go @@ -0,0 +1,2061 @@ +// Copyright (c) 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Inital yacc source generated by ebnf2y[1] +// at 2013-10-04 23:10:47.861401015 +0200 CEST +// +// $ ebnf2y -o ql.y -oe ql.ebnf -start StatementList -pkg ql -p _ +// +// CAUTION: If this file is a Go source file (*.go), it was generated +// automatically by '$ goyacc' from a *.y file - DO NOT EDIT in that case! +// +// [1]: http://github.com/cznic/ebnf2y + +package ql + +import __yyfmt__ "fmt" + +import ( + "fmt" + + "github.com/cznic/mathutil" +) + +type yySymType struct { + yys int + line int + col int + item interface{} + list []interface{} +} + +type yyXError struct { + state, xsym int +} + +const ( + yyDefault = 57429 + yyEOFCode = 57344 + add = 57346 + alter = 57347 + and = 57348 + andand = 57349 + andnot = 57350 + as = 57351 + asc = 57352 + begin = 57353 + between = 57354 + bigIntType = 57355 + bigRatType = 57356 + blobType = 57357 + boolType = 57358 + by = 57359 + byteType = 57360 + column = 57361 + commit = 57362 + complex128Type = 57363 + complex64Type = 57364 + create = 57365 + deleteKwd = 57366 + desc = 57367 + distinct = 57368 + drop = 57369 + durationType = 57370 + eq = 57371 + yyErrCode = 57345 + exists = 57372 + falseKwd = 57373 + float32Type = 57375 + float64Type = 57376 + floatLit = 57377 + floatType = 57374 + from = 57378 + ge = 57379 + group = 57380 + identifier = 57381 + ifKwd = 57382 + imaginaryLit = 57383 + in = 57384 + index = 57385 + insert = 57386 + int16Type = 57388 + int32Type = 57389 + int64Type = 57390 + int8Type = 57391 + intLit = 57393 + intType = 57387 + into = 57392 + is = 57394 + le = 57395 + like = 57396 + limit = 57397 + lsh = 57398 + neq = 57399 + not = 57400 + null = 57401 + offset = 57402 + on = 57403 + or = 57404 + order = 57405 + oror = 57406 + qlParam = 57407 + rollback = 57408 + rsh = 57409 + runeType = 57410 + selectKwd = 57411 + set = 57412 + stringLit = 57414 + stringType = 57413 + tableKwd = 57415 + timeType = 57416 + transaction = 57417 + trueKwd = 57418 + truncate = 57419 + uint16Type = 57421 + uint32Type = 57422 + uint64Type = 57423 + uint8Type = 57424 + uintType = 57420 + unique = 57425 + update = 57426 + values = 57427 + where = 57428 + + yyMaxDepth = 200 + yyTabOfs = -206 +) + +var ( + yyXLAT = map[int]int{ + 59: 0, // ';' (181x) + 57344: 1, // $end (180x) + 41: 2, // ')' (156x) + 44: 3, // ',' (122x) + 40: 4, // '(' (119x) + 43: 5, // '+' (103x) + 45: 6, // '-' (103x) + 94: 7, // '^' (103x) + 57402: 8, // offset (102x) + 57397: 9, // limit (99x) + 57381: 10, // identifier (87x) + 57405: 11, // order (87x) + 57428: 12, // where (82x) + 57380: 13, // group (77x) + 57404: 14, // or (76x) + 57406: 15, // oror (76x) + 57378: 16, // from (74x) + 57352: 17, // asc (70x) + 57367: 18, // desc (70x) + 93: 19, // ']' (69x) + 57351: 20, // as (68x) + 58: 21, // ':' (66x) + 57348: 22, // and (66x) + 57349: 23, // andand (64x) + 124: 24, // '|' (55x) + 57400: 25, // not (55x) + 57355: 26, // bigIntType (54x) + 57356: 27, // bigRatType (54x) + 57357: 28, // blobType (54x) + 57358: 29, // boolType (54x) + 57360: 30, // byteType (54x) + 57363: 31, // complex128Type (54x) + 57364: 32, // complex64Type (54x) + 57370: 33, // durationType (54x) + 57375: 34, // float32Type (54x) + 57376: 35, // float64Type (54x) + 57374: 36, // floatType (54x) + 57388: 37, // int16Type (54x) + 57389: 38, // int32Type (54x) + 57390: 39, // int64Type (54x) + 57391: 40, // int8Type (54x) + 57387: 41, // intType (54x) + 57401: 42, // null (54x) + 57410: 43, // runeType (54x) + 57413: 44, // stringType (54x) + 57416: 45, // timeType (54x) + 57421: 46, // uint16Type (54x) + 57422: 47, // uint32Type (54x) + 57423: 48, // uint64Type (54x) + 57424: 49, // uint8Type (54x) + 57420: 50, // uintType (54x) + 57354: 51, // between (53x) + 57384: 52, // in (53x) + 60: 53, // '<' (52x) + 62: 54, // '>' (52x) + 57371: 55, // eq (52x) + 57373: 56, // falseKwd (52x) + 57377: 57, // floatLit (52x) + 57379: 58, // ge (52x) + 57383: 59, // imaginaryLit (52x) + 57393: 60, // intLit (52x) + 57394: 61, // is (52x) + 57395: 62, // le (52x) + 57396: 63, // like (52x) + 57399: 64, // neq (52x) + 57407: 65, // qlParam (52x) + 57414: 66, // stringLit (52x) + 57418: 67, // trueKwd (52x) + 33: 68, // '!' (48x) + 57499: 69, // Type (47x) + 42: 70, // '*' (46x) + 57444: 71, // Conversion (46x) + 57471: 72, // Literal (46x) + 57472: 73, // Operand (46x) + 57475: 74, // PrimaryExpression (46x) + 57478: 75, // QualifiedIdent (46x) + 37: 76, // '%' (43x) + 38: 77, // '&' (43x) + 47: 78, // '/' (43x) + 57350: 79, // andnot (43x) + 57398: 80, // lsh (43x) + 57409: 81, // rsh (43x) + 57500: 82, // UnaryExpr (42x) + 57477: 83, // PrimaryTerm (35x) + 57476: 84, // PrimaryFactor (31x) + 91: 85, // '[' (30x) + 57460: 86, // Factor (20x) + 57461: 87, // Factor1 (20x) + 57497: 88, // Term (19x) + 57456: 89, // Expression (18x) + 57505: 90, // logOr (12x) + 57439: 91, // ColumnName (10x) + 57411: 92, // selectKwd (9x) + 57496: 93, // TableName (9x) + 57457: 94, // ExpressionList (6x) + 57485: 95, // SelectStmt (6x) + 57436: 96, // Call (5x) + 57466: 97, // Index (5x) + 57493: 98, // Slice (5x) + 57438: 99, // ColumnDef (4x) + 57369: 100, // drop (4x) + 57372: 101, // exists (4x) + 57382: 102, // ifKwd (4x) + 57385: 103, // index (4x) + 57415: 104, // tableKwd (4x) + 57427: 105, // values (4x) + 57503: 106, // WhereClause (4x) + 61: 107, // '=' (2x) + 57346: 108, // add (2x) + 57347: 109, // alter (2x) + 57430: 110, // AlterTableStmt (2x) + 57431: 111, // Assignment (2x) + 57353: 112, // begin (2x) + 57435: 113, // BeginTransactionStmt (2x) + 57359: 114, // by (2x) + 57440: 115, // ColumnNameList (2x) + 57362: 116, // commit (2x) + 57443: 117, // CommitStmt (2x) + 57365: 118, // create (2x) + 57446: 119, // CreateIndexStmt (2x) + 57448: 120, // CreateTableStmt (2x) + 57449: 121, // CreateTableStmt1 (2x) + 57450: 122, // CreateTableStmt2 (2x) + 57451: 123, // DeleteFromStmt (2x) + 57366: 124, // deleteKwd (2x) + 57453: 125, // DropIndexStmt (2x) + 57454: 126, // DropTableStmt (2x) + 57455: 127, // EmptyStmt (2x) + 57462: 128, // Field (2x) + 57465: 129, // GroupByClause (2x) + 57386: 130, // insert (2x) + 57467: 131, // InsertIntoStmt (2x) + 57504: 132, // logAnd (2x) + 57473: 133, // OrderBy (2x) + 57479: 134, // RecordSet (2x) + 57480: 135, // RecordSet1 (2x) + 57408: 136, // rollback (2x) + 57484: 137, // RollbackStmt (2x) + 57488: 138, // SelectStmtGroup (2x) + 57489: 139, // SelectStmtLimit (2x) + 57490: 140, // SelectStmtOffset (2x) + 57491: 141, // SelectStmtOrder (2x) + 57492: 142, // SelectStmtWhere (2x) + 57412: 143, // set (2x) + 57494: 144, // Statement (2x) + 57419: 145, // truncate (2x) + 57498: 146, // TruncateTableStmt (2x) + 57426: 147, // update (2x) + 57501: 148, // UpdateStmt (2x) + 46: 149, // '.' (1x) + 57432: 150, // AssignmentList (1x) + 57433: 151, // AssignmentList1 (1x) + 57434: 152, // AssignmentList2 (1x) + 57437: 153, // Call1 (1x) + 57361: 154, // column (1x) + 57441: 155, // ColumnNameList1 (1x) + 57442: 156, // ColumnNameList2 (1x) + 57445: 157, // CreateIndexIfNotExists (1x) + 57447: 158, // CreateIndexStmtUnique (1x) + 57368: 159, // distinct (1x) + 57452: 160, // DropIndexIfExists (1x) + 57458: 161, // ExpressionList1 (1x) + 57459: 162, // ExpressionList2 (1x) + 57463: 163, // Field1 (1x) + 57464: 164, // FieldList (1x) + 57468: 165, // InsertIntoStmt1 (1x) + 57469: 166, // InsertIntoStmt2 (1x) + 57470: 167, // InsertIntoStmt3 (1x) + 57392: 168, // into (1x) + 57403: 169, // on (1x) + 57474: 170, // OrderBy1 (1x) + 57506: 171, // oSet (1x) + 57481: 172, // RecordSet11 (1x) + 57482: 173, // RecordSet2 (1x) + 57483: 174, // RecordSetList (1x) + 57486: 175, // SelectStmtDistinct (1x) + 57487: 176, // SelectStmtFieldList (1x) + 57495: 177, // StatementList (1x) + 57417: 178, // transaction (1x) + 57425: 179, // unique (1x) + 57502: 180, // UpdateStmt1 (1x) + 57429: 181, // $default (0x) + 57345: 182, // error (0x) + } + + yySymNames = []string{ + "';'", + "$end", + "')'", + "','", + "'('", + "'+'", + "'-'", + "'^'", + "offset", + "limit", + "identifier", + "order", + "where", + "group", + "or", + "oror", + "from", + "asc", + "desc", + "']'", + "as", + "':'", + "and", + "andand", + "'|'", + "not", + "bigIntType", + "bigRatType", + "blobType", + "boolType", + "byteType", + "complex128Type", + "complex64Type", + "durationType", + "float32Type", + "float64Type", + "floatType", + "int16Type", + "int32Type", + "int64Type", + "int8Type", + "intType", + "null", + "runeType", + "stringType", + "timeType", + "uint16Type", + "uint32Type", + "uint64Type", + "uint8Type", + "uintType", + "between", + "in", + "'<'", + "'>'", + "eq", + "falseKwd", + "floatLit", + "ge", + "imaginaryLit", + "intLit", + "is", + "le", + "like", + "neq", + "qlParam", + "stringLit", + "trueKwd", + "'!'", + "Type", + "'*'", + "Conversion", + "Literal", + "Operand", + "PrimaryExpression", + "QualifiedIdent", + "'%'", + "'&'", + "'/'", + "andnot", + "lsh", + "rsh", + "UnaryExpr", + "PrimaryTerm", + "PrimaryFactor", + "'['", + "Factor", + "Factor1", + "Term", + "Expression", + "logOr", + "ColumnName", + "selectKwd", + "TableName", + "ExpressionList", + "SelectStmt", + "Call", + "Index", + "Slice", + "ColumnDef", + "drop", + "exists", + "ifKwd", + "index", + "tableKwd", + "values", + "WhereClause", + "'='", + "add", + "alter", + "AlterTableStmt", + "Assignment", + "begin", + "BeginTransactionStmt", + "by", + "ColumnNameList", + "commit", + "CommitStmt", + "create", + "CreateIndexStmt", + "CreateTableStmt", + "CreateTableStmt1", + "CreateTableStmt2", + "DeleteFromStmt", + "deleteKwd", + "DropIndexStmt", + "DropTableStmt", + "EmptyStmt", + "Field", + "GroupByClause", + "insert", + "InsertIntoStmt", + "logAnd", + "OrderBy", + "RecordSet", + "RecordSet1", + "rollback", + "RollbackStmt", + "SelectStmtGroup", + "SelectStmtLimit", + "SelectStmtOffset", + "SelectStmtOrder", + "SelectStmtWhere", + "set", + "Statement", + "truncate", + "TruncateTableStmt", + "update", + "UpdateStmt", + "'.'", + "AssignmentList", + "AssignmentList1", + "AssignmentList2", + "Call1", + "column", + "ColumnNameList1", + "ColumnNameList2", + "CreateIndexIfNotExists", + "CreateIndexStmtUnique", + "distinct", + "DropIndexIfExists", + "ExpressionList1", + "ExpressionList2", + "Field1", + "FieldList", + "InsertIntoStmt1", + "InsertIntoStmt2", + "InsertIntoStmt3", + "into", + "on", + "OrderBy1", + "oSet", + "RecordSet11", + "RecordSet2", + "RecordSetList", + "SelectStmtDistinct", + "SelectStmtFieldList", + "StatementList", + "transaction", + "unique", + "UpdateStmt1", + "$default", + "error", + } + + yyReductions = map[int]struct{ xsym, components int }{ + 0: {0, 1}, + 1: {110, 5}, + 2: {110, 6}, + 3: {111, 3}, + 4: {150, 3}, + 5: {151, 0}, + 6: {151, 3}, + 7: {152, 0}, + 8: {152, 1}, + 9: {113, 2}, + 10: {96, 3}, + 11: {153, 0}, + 12: {153, 1}, + 13: {99, 2}, + 14: {91, 1}, + 15: {115, 3}, + 16: {155, 0}, + 17: {155, 3}, + 18: {156, 0}, + 19: {156, 1}, + 20: {117, 1}, + 21: {71, 4}, + 22: {119, 10}, + 23: {119, 12}, + 24: {157, 0}, + 25: {157, 3}, + 26: {158, 0}, + 27: {158, 1}, + 28: {120, 8}, + 29: {120, 11}, + 30: {121, 0}, + 31: {121, 3}, + 32: {122, 0}, + 33: {122, 1}, + 34: {123, 3}, + 35: {123, 4}, + 36: {125, 4}, + 37: {160, 0}, + 38: {160, 2}, + 39: {126, 3}, + 40: {126, 5}, + 41: {127, 0}, + 42: {89, 1}, + 43: {89, 3}, + 44: {90, 1}, + 45: {90, 1}, + 46: {94, 3}, + 47: {161, 0}, + 48: {161, 3}, + 49: {162, 0}, + 50: {162, 1}, + 51: {86, 1}, + 52: {86, 5}, + 53: {86, 6}, + 54: {86, 5}, + 55: {86, 6}, + 56: {86, 5}, + 57: {86, 6}, + 58: {86, 3}, + 59: {86, 4}, + 60: {87, 1}, + 61: {87, 3}, + 62: {87, 3}, + 63: {87, 3}, + 64: {87, 3}, + 65: {87, 3}, + 66: {87, 3}, + 67: {87, 3}, + 68: {128, 2}, + 69: {163, 0}, + 70: {163, 2}, + 71: {164, 1}, + 72: {164, 3}, + 73: {129, 3}, + 74: {97, 3}, + 75: {131, 10}, + 76: {131, 5}, + 77: {165, 0}, + 78: {165, 3}, + 79: {166, 0}, + 80: {166, 5}, + 81: {167, 0}, + 82: {167, 1}, + 83: {72, 1}, + 84: {72, 1}, + 85: {72, 1}, + 86: {72, 1}, + 87: {72, 1}, + 88: {72, 1}, + 89: {72, 1}, + 90: {73, 1}, + 91: {73, 1}, + 92: {73, 1}, + 93: {73, 3}, + 94: {133, 4}, + 95: {170, 0}, + 96: {170, 1}, + 97: {170, 1}, + 98: {74, 1}, + 99: {74, 1}, + 100: {74, 2}, + 101: {74, 2}, + 102: {74, 2}, + 103: {84, 1}, + 104: {84, 3}, + 105: {84, 3}, + 106: {84, 3}, + 107: {84, 3}, + 108: {83, 1}, + 109: {83, 3}, + 110: {83, 3}, + 111: {83, 3}, + 112: {83, 3}, + 113: {83, 3}, + 114: {83, 3}, + 115: {83, 3}, + 116: {75, 1}, + 117: {75, 3}, + 118: {134, 2}, + 119: {135, 1}, + 120: {135, 4}, + 121: {172, 0}, + 122: {172, 1}, + 123: {173, 0}, + 124: {173, 2}, + 125: {174, 1}, + 126: {174, 3}, + 127: {137, 1}, + 128: {95, 10}, + 129: {95, 11}, + 130: {139, 0}, + 131: {139, 2}, + 132: {140, 0}, + 133: {140, 2}, + 134: {175, 0}, + 135: {175, 1}, + 136: {176, 1}, + 137: {176, 1}, + 138: {176, 2}, + 139: {142, 0}, + 140: {142, 1}, + 141: {138, 0}, + 142: {138, 1}, + 143: {141, 0}, + 144: {141, 1}, + 145: {98, 3}, + 146: {98, 4}, + 147: {98, 4}, + 148: {98, 5}, + 149: {144, 1}, + 150: {144, 1}, + 151: {144, 1}, + 152: {144, 1}, + 153: {144, 1}, + 154: {144, 1}, + 155: {144, 1}, + 156: {144, 1}, + 157: {144, 1}, + 158: {144, 1}, + 159: {144, 1}, + 160: {144, 1}, + 161: {144, 1}, + 162: {144, 1}, + 163: {177, 1}, + 164: {177, 3}, + 165: {93, 1}, + 166: {88, 1}, + 167: {88, 3}, + 168: {132, 1}, + 169: {132, 1}, + 170: {146, 3}, + 171: {69, 1}, + 172: {69, 1}, + 173: {69, 1}, + 174: {69, 1}, + 175: {69, 1}, + 176: {69, 1}, + 177: {69, 1}, + 178: {69, 1}, + 179: {69, 1}, + 180: {69, 1}, + 181: {69, 1}, + 182: {69, 1}, + 183: {69, 1}, + 184: {69, 1}, + 185: {69, 1}, + 186: {69, 1}, + 187: {69, 1}, + 188: {69, 1}, + 189: {69, 1}, + 190: {69, 1}, + 191: {69, 1}, + 192: {69, 1}, + 193: {69, 1}, + 194: {69, 1}, + 195: {148, 5}, + 196: {180, 0}, + 197: {180, 1}, + 198: {82, 1}, + 199: {82, 2}, + 200: {82, 2}, + 201: {82, 2}, + 202: {82, 2}, + 203: {106, 2}, + 204: {171, 0}, + 205: {171, 1}, + } + + yyXErrors = map[yyXError]string{} + + yyParseTab = [335][]uint16{ + // 0 + {165, 165, 92: 216, 95: 228, 100: 213, 109: 208, 218, 112: 209, 219, 116: 210, 220, 211, 221, 222, 123: 223, 212, 224, 225, 217, 130: 214, 226, 136: 215, 227, 144: 231, 232, 229, 233, 230, 177: 207}, + {539, 206}, + {104: 532}, + {178: 531}, + {186, 186}, + // 5 + {103: 180, 497, 158: 495, 179: 496}, + {16: 492}, + {103: 482, 483}, + {168: 465}, + {79, 79}, + // 10 + {4: 72, 72, 72, 72, 10: 72, 26: 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 56: 72, 72, 59: 72, 72, 65: 72, 72, 72, 72, 70: 72, 159: 408, 175: 407}, + {57, 57}, + {56, 56}, + {55, 55}, + {54, 54}, + // 15 + {53, 53}, + {52, 52}, + {51, 51}, + {50, 50}, + {49, 49}, + // 20 + {48, 48}, + {47, 47}, + {46, 46}, + {45, 45}, + {44, 44}, + // 25 + {43, 43}, + {104: 405}, + {10: 234, 93: 235}, + {41, 41, 4: 41, 10: 41, 12: 41, 92: 41, 100: 41, 105: 41, 108: 41, 143: 41}, + {10: 2, 143: 237, 171: 236}, + // 30 + {10: 240, 91: 238, 111: 239, 150: 241}, + {10: 1}, + {107: 403}, + {201, 201, 3: 201, 12: 201, 151: 399}, + {192, 192, 192, 192, 8: 192, 192, 11: 192, 26: 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 43: 192, 192, 192, 192, 192, 192, 192, 192, 107: 192}, + // 35 + {10, 10, 12: 244, 106: 243, 180: 242}, + {11, 11}, + {9, 9}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 247}, + {4: 396}, + // 40 + {164, 164, 164, 164, 8: 164, 164, 11: 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 313, 312, 132: 311}, + {3, 3, 3, 8: 3, 3, 11: 3, 13: 3, 308, 307, 90: 306}, + {155, 155, 155, 155, 8: 155, 155, 11: 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 25: 358, 51: 359, 357, 364, 362, 366, 58: 361, 61: 360, 363, 367, 365}, + {146, 146, 146, 146, 5: 352, 351, 349, 146, 146, 11: 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 350, 146, 51: 146, 146, 146, 146, 146, 58: 146, 61: 146, 146, 146, 146}, + {123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 11: 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 51: 123, 123, 123, 123, 123, 58: 123, 61: 123, 123, 123, 123, 70: 123, 76: 123, 123, 123, 123, 123, 123, 85: 123}, + // 45 + {122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 11: 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 51: 122, 122, 122, 122, 122, 58: 122, 61: 122, 122, 122, 122, 70: 122, 76: 122, 122, 122, 122, 122, 122, 85: 122}, + {121, 121, 121, 121, 121, 121, 121, 121, 121, 121, 11: 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, 51: 121, 121, 121, 121, 121, 58: 121, 61: 121, 121, 121, 121, 70: 121, 76: 121, 121, 121, 121, 121, 121, 85: 121}, + {120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 11: 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 51: 120, 120, 120, 120, 120, 58: 120, 61: 120, 120, 120, 120, 70: 120, 76: 120, 120, 120, 120, 120, 120, 85: 120}, + {119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 11: 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 51: 119, 119, 119, 119, 119, 58: 119, 61: 119, 119, 119, 119, 70: 119, 76: 119, 119, 119, 119, 119, 119, 85: 119}, + {118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 11: 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 51: 118, 118, 118, 118, 118, 58: 118, 61: 118, 118, 118, 118, 70: 118, 76: 118, 118, 118, 118, 118, 118, 85: 118}, + // 50 + {117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 11: 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 51: 117, 117, 117, 117, 117, 58: 117, 61: 117, 117, 117, 117, 70: 117, 76: 117, 117, 117, 117, 117, 117, 85: 117}, + {116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 11: 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 51: 116, 116, 116, 116, 116, 58: 116, 61: 116, 116, 116, 116, 70: 116, 76: 116, 116, 116, 116, 116, 116, 85: 116}, + {115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 11: 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 51: 115, 115, 115, 115, 115, 58: 115, 61: 115, 115, 115, 115, 70: 115, 76: 115, 115, 115, 115, 115, 115, 85: 115}, + {114, 114, 114, 114, 114, 114, 114, 114, 114, 114, 11: 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, 51: 114, 114, 114, 114, 114, 58: 114, 61: 114, 114, 114, 114, 70: 114, 76: 114, 114, 114, 114, 114, 114, 85: 114}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 347}, + // 55 + {108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 11: 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 51: 108, 108, 108, 108, 108, 58: 108, 61: 108, 108, 108, 108, 70: 108, 76: 108, 108, 108, 108, 108, 108, 85: 108}, + {107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 11: 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 51: 107, 107, 107, 107, 107, 58: 107, 61: 107, 107, 107, 107, 70: 107, 76: 107, 107, 107, 107, 107, 107, 85: 107}, + {8, 8, 8, 8, 297, 8, 8, 8, 8, 8, 11: 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 51: 8, 8, 8, 8, 8, 58: 8, 61: 8, 8, 8, 8, 70: 8, 76: 8, 8, 8, 8, 8, 8, 85: 298, 96: 301, 299, 300}, + {103, 103, 103, 103, 5: 103, 103, 103, 103, 103, 11: 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 51: 103, 103, 103, 103, 103, 58: 103, 61: 103, 103, 103, 103, 70: 339, 76: 337, 334, 338, 333, 335, 336}, + {98, 98, 98, 98, 5: 98, 98, 98, 98, 98, 11: 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 51: 98, 98, 98, 98, 98, 58: 98, 61: 98, 98, 98, 98, 70: 98, 76: 98, 98, 98, 98, 98, 98}, + // 60 + {90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 11: 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, 51: 90, 90, 90, 90, 90, 58: 90, 61: 90, 90, 90, 90, 70: 90, 76: 90, 90, 90, 90, 90, 90, 85: 90, 149: 331}, + {40, 40, 40, 40, 8: 40, 40, 11: 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40}, + {35, 35, 35, 35, 35}, + {34, 34, 34, 34, 34}, + {33, 33, 33, 33, 33}, + // 65 + {32, 32, 32, 32, 32}, + {31, 31, 31, 31, 31}, + {30, 30, 30, 30, 30}, + {29, 29, 29, 29, 29}, + {28, 28, 28, 28, 28}, + // 70 + {27, 27, 27, 27, 27}, + {26, 26, 26, 26, 26}, + {25, 25, 25, 25, 25}, + {24, 24, 24, 24, 24}, + {23, 23, 23, 23, 23}, + // 75 + {22, 22, 22, 22, 22}, + {21, 21, 21, 21, 21}, + {20, 20, 20, 20, 20}, + {19, 19, 19, 19, 19}, + {18, 18, 18, 18, 18}, + // 80 + {17, 17, 17, 17, 17}, + {16, 16, 16, 16, 16}, + {15, 15, 15, 15, 15}, + {14, 14, 14, 14, 14}, + {13, 13, 13, 13, 13}, + // 85 + {12, 12, 12, 12, 12}, + {4: 260, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 69: 245, 71: 262, 257, 261, 330, 259}, + {4: 260, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 69: 245, 71: 262, 257, 261, 329, 259}, + {4: 260, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 69: 245, 71: 262, 257, 261, 328, 259}, + {4: 260, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 69: 245, 71: 262, 257, 261, 296, 259}, + // 90 + {4, 4, 4, 4, 297, 4, 4, 4, 4, 4, 11: 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 51: 4, 4, 4, 4, 4, 58: 4, 61: 4, 4, 4, 4, 70: 4, 76: 4, 4, 4, 4, 4, 4, 85: 298, 96: 301, 299, 300}, + {2: 195, 4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 322, 94: 321, 153: 320}, + {4: 260, 295, 294, 292, 10: 266, 21: 303, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 302}, + {106, 106, 106, 106, 106, 106, 106, 106, 106, 106, 11: 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, 51: 106, 106, 106, 106, 106, 58: 106, 61: 106, 106, 106, 106, 70: 106, 76: 106, 106, 106, 106, 106, 106, 85: 106}, + {105, 105, 105, 105, 105, 105, 105, 105, 105, 105, 11: 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, 51: 105, 105, 105, 105, 105, 58: 105, 61: 105, 105, 105, 105, 70: 105, 76: 105, 105, 105, 105, 105, 105, 85: 105}, + // 95 + {104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 11: 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 51: 104, 104, 104, 104, 104, 58: 104, 61: 104, 104, 104, 104, 70: 104, 76: 104, 104, 104, 104, 104, 104, 85: 104}, + {14: 308, 307, 19: 315, 21: 316, 90: 306}, + {4: 260, 295, 294, 292, 10: 266, 19: 305, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 304}, + {14: 308, 307, 19: 309, 90: 306}, + {61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 11: 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 51: 61, 61, 61, 61, 61, 58: 61, 61: 61, 61, 61, 61, 70: 61, 76: 61, 61, 61, 61, 61, 61, 85: 61}, + // 100 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 310}, + {4: 162, 162, 162, 162, 10: 162, 26: 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 56: 162, 162, 59: 162, 162, 65: 162, 162, 162, 162}, + {4: 161, 161, 161, 161, 10: 161, 26: 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, 56: 161, 161, 59: 161, 161, 65: 161, 161, 161, 161}, + {60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 11: 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 51: 60, 60, 60, 60, 60, 58: 60, 61: 60, 60, 60, 60, 70: 60, 76: 60, 60, 60, 60, 60, 60, 85: 60}, + {163, 163, 163, 163, 8: 163, 163, 11: 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 163, 313, 312, 132: 311}, + // 105 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 314, 248}, + {4: 38, 38, 38, 38, 10: 38, 26: 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 56: 38, 38, 59: 38, 38, 65: 38, 38, 38, 38}, + {4: 37, 37, 37, 37, 10: 37, 26: 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 56: 37, 37, 59: 37, 37, 65: 37, 37, 37, 37}, + {39, 39, 39, 39, 8: 39, 39, 11: 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39}, + {132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 11: 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 51: 132, 132, 132, 132, 132, 58: 132, 61: 132, 132, 132, 132, 70: 132, 76: 132, 132, 132, 132, 132, 132, 85: 132}, + // 110 + {4: 260, 295, 294, 292, 10: 266, 19: 318, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 317}, + {14: 308, 307, 19: 319, 90: 306}, + {59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 11: 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 51: 59, 59, 59, 59, 59, 58: 59, 61: 59, 59, 59, 59, 70: 59, 76: 59, 59, 59, 59, 59, 59, 85: 59}, + {58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 11: 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 51: 58, 58, 58, 58, 58, 58: 58, 61: 58, 58, 58, 58, 70: 58, 76: 58, 58, 58, 58, 58, 58, 85: 58}, + {2: 327}, + // 115 + {2: 194}, + {159, 159, 159, 159, 8: 159, 159, 14: 308, 307, 17: 159, 159, 90: 306, 161: 323}, + {157, 157, 157, 325, 8: 157, 157, 17: 157, 157, 162: 324}, + {160, 160, 160, 8: 160, 160, 17: 160, 160}, + {156, 156, 156, 4: 260, 295, 294, 292, 156, 156, 266, 17: 156, 156, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 326}, + // 120 + {158, 158, 158, 158, 8: 158, 158, 14: 308, 307, 17: 158, 158, 90: 306}, + {196, 196, 196, 196, 196, 196, 196, 196, 196, 196, 11: 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, 51: 196, 196, 196, 196, 196, 58: 196, 61: 196, 196, 196, 196, 70: 196, 76: 196, 196, 196, 196, 196, 196, 85: 196}, + {5, 5, 5, 5, 297, 5, 5, 5, 5, 5, 11: 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 51: 5, 5, 5, 5, 5, 58: 5, 61: 5, 5, 5, 5, 70: 5, 76: 5, 5, 5, 5, 5, 5, 85: 298, 96: 301, 299, 300}, + {6, 6, 6, 6, 297, 6, 6, 6, 6, 6, 11: 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 51: 6, 6, 6, 6, 6, 58: 6, 61: 6, 6, 6, 6, 70: 6, 76: 6, 6, 6, 6, 6, 6, 85: 298, 96: 301, 299, 300}, + {7, 7, 7, 7, 297, 7, 7, 7, 7, 7, 11: 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 51: 7, 7, 7, 7, 7, 58: 7, 61: 7, 7, 7, 7, 70: 7, 76: 7, 7, 7, 7, 7, 7, 85: 298, 96: 301, 299, 300}, + // 125 + {10: 332}, + {89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 11: 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 51: 89, 89, 89, 89, 89, 58: 89, 61: 89, 89, 89, 89, 70: 89, 76: 89, 89, 89, 89, 89, 89, 85: 89}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 346}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 345}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 344}, + // 130 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 343}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 342}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 341}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 340}, + {91, 91, 91, 91, 5: 91, 91, 91, 91, 91, 11: 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 51: 91, 91, 91, 91, 91, 58: 91, 61: 91, 91, 91, 91, 70: 91, 76: 91, 91, 91, 91, 91, 91}, + // 135 + {92, 92, 92, 92, 5: 92, 92, 92, 92, 92, 11: 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 51: 92, 92, 92, 92, 92, 58: 92, 61: 92, 92, 92, 92, 70: 92, 76: 92, 92, 92, 92, 92, 92}, + {93, 93, 93, 93, 5: 93, 93, 93, 93, 93, 11: 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, 93, 51: 93, 93, 93, 93, 93, 58: 93, 61: 93, 93, 93, 93, 70: 93, 76: 93, 93, 93, 93, 93, 93}, + {94, 94, 94, 94, 5: 94, 94, 94, 94, 94, 11: 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 51: 94, 94, 94, 94, 94, 58: 94, 61: 94, 94, 94, 94, 70: 94, 76: 94, 94, 94, 94, 94, 94}, + {95, 95, 95, 95, 5: 95, 95, 95, 95, 95, 11: 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 51: 95, 95, 95, 95, 95, 58: 95, 61: 95, 95, 95, 95, 70: 95, 76: 95, 95, 95, 95, 95, 95}, + {96, 96, 96, 96, 5: 96, 96, 96, 96, 96, 11: 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 51: 96, 96, 96, 96, 96, 58: 96, 61: 96, 96, 96, 96, 70: 96, 76: 96, 96, 96, 96, 96, 96}, + // 140 + {97, 97, 97, 97, 5: 97, 97, 97, 97, 97, 11: 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 51: 97, 97, 97, 97, 97, 58: 97, 61: 97, 97, 97, 97, 70: 97, 76: 97, 97, 97, 97, 97, 97}, + {2: 348, 14: 308, 307, 90: 306}, + {113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 11: 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 51: 113, 113, 113, 113, 113, 58: 113, 61: 113, 113, 113, 113, 70: 113, 76: 113, 113, 113, 113, 113, 113, 85: 113}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 356}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 355}, + // 145 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 354}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 353}, + {99, 99, 99, 99, 5: 99, 99, 99, 99, 99, 11: 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 51: 99, 99, 99, 99, 99, 58: 99, 61: 99, 99, 99, 99, 70: 339, 76: 337, 334, 338, 333, 335, 336}, + {100, 100, 100, 100, 5: 100, 100, 100, 100, 100, 11: 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 51: 100, 100, 100, 100, 100, 58: 100, 61: 100, 100, 100, 100, 70: 339, 76: 337, 334, 338, 333, 335, 336}, + {101, 101, 101, 101, 5: 101, 101, 101, 101, 101, 11: 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 51: 101, 101, 101, 101, 101, 58: 101, 61: 101, 101, 101, 101, 70: 339, 76: 337, 334, 338, 333, 335, 336}, + // 150 + {102, 102, 102, 102, 5: 102, 102, 102, 102, 102, 11: 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 51: 102, 102, 102, 102, 102, 58: 102, 61: 102, 102, 102, 102, 70: 339, 76: 337, 334, 338, 333, 335, 336}, + {4: 391}, + {51: 382, 381}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 378}, + {25: 376, 42: 375}, + // 155 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 374}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 373}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 372}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 371}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 370}, + // 160 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 369}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 368}, + {139, 139, 139, 139, 5: 352, 351, 349, 139, 139, 11: 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 350, 139, 51: 139, 139, 139, 139, 139, 58: 139, 61: 139, 139, 139, 139}, + {140, 140, 140, 140, 5: 352, 351, 349, 140, 140, 11: 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 350, 140, 51: 140, 140, 140, 140, 140, 58: 140, 61: 140, 140, 140, 140}, + {141, 141, 141, 141, 5: 352, 351, 349, 141, 141, 11: 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 350, 141, 51: 141, 141, 141, 141, 141, 58: 141, 61: 141, 141, 141, 141}, + // 165 + {142, 142, 142, 142, 5: 352, 351, 349, 142, 142, 11: 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 350, 142, 51: 142, 142, 142, 142, 142, 58: 142, 61: 142, 142, 142, 142}, + {143, 143, 143, 143, 5: 352, 351, 349, 143, 143, 11: 143, 143, 143, 143, 143, 143, 143, 143, 143, 143, 143, 143, 143, 350, 143, 51: 143, 143, 143, 143, 143, 58: 143, 61: 143, 143, 143, 143}, + {144, 144, 144, 144, 5: 352, 351, 349, 144, 144, 11: 144, 144, 144, 144, 144, 144, 144, 144, 144, 144, 144, 144, 144, 350, 144, 51: 144, 144, 144, 144, 144, 58: 144, 61: 144, 144, 144, 144}, + {145, 145, 145, 145, 5: 352, 351, 349, 145, 145, 11: 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, 350, 145, 51: 145, 145, 145, 145, 145, 58: 145, 61: 145, 145, 145, 145}, + {148, 148, 148, 148, 8: 148, 148, 11: 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148}, + // 170 + {42: 377}, + {147, 147, 147, 147, 8: 147, 147, 11: 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147, 147}, + {5: 352, 351, 349, 22: 379, 24: 350}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 380}, + {150, 150, 150, 150, 5: 352, 351, 349, 150, 150, 11: 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 350}, + // 175 + {4: 386}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 383}, + {5: 352, 351, 349, 22: 384, 24: 350}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 385}, + {149, 149, 149, 149, 5: 352, 351, 349, 149, 149, 11: 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 149, 350}, + // 180 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 322, 92: 216, 94: 387, 388}, + {2: 390}, + {2: 389}, + {151, 151, 151, 151, 8: 151, 151, 11: 151, 151, 151, 151, 151, 151, 151, 151, 151, 151, 151, 151, 151}, + {153, 153, 153, 153, 8: 153, 153, 11: 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153}, + // 185 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 322, 92: 216, 94: 392, 393}, + {2: 395}, + {2: 394}, + {152, 152, 152, 152, 8: 152, 152, 11: 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152}, + {154, 154, 154, 154, 8: 154, 154, 11: 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, 154}, + // 190 + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 397}, + {2: 398, 14: 308, 307, 90: 306}, + {185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 11: 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 51: 185, 185, 185, 185, 185, 58: 185, 61: 185, 185, 185, 185, 70: 185, 76: 185, 185, 185, 185, 185, 185, 85: 185}, + {199, 199, 3: 401, 12: 199, 152: 400}, + {202, 202, 12: 202}, + // 195 + {198, 198, 10: 240, 12: 198, 91: 238, 111: 402}, + {200, 200, 3: 200, 12: 200}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 404}, + {203, 203, 3: 203, 12: 203, 14: 308, 307, 90: 306}, + {10: 234, 93: 406}, + // 200 + {36, 36}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 413, 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 409, 128: 410, 164: 411, 176: 412}, + {4: 71, 71, 71, 71, 10: 71, 26: 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 56: 71, 71, 59: 71, 71, 65: 71, 71, 71, 71, 70: 71}, + {3: 137, 14: 308, 307, 137, 20: 463, 90: 306, 163: 462}, + {3: 135, 16: 135}, + // 205 + {3: 460, 16: 69}, + {16: 414}, + {16: 70}, + {4: 417, 10: 416, 134: 418, 415, 174: 419}, + {83, 83, 83, 83, 8: 83, 83, 11: 83, 83, 83, 20: 458, 173: 457}, + // 210 + {87, 87, 87, 87, 8: 87, 87, 11: 87, 87, 87, 20: 87}, + {92: 216, 95: 453}, + {81, 81, 81, 81, 8: 81, 81, 11: 81, 81, 81}, + {67, 67, 67, 420, 8: 67, 67, 11: 67, 244, 67, 106: 422, 142: 421}, + {67, 67, 67, 4: 417, 8: 67, 67, 416, 67, 244, 67, 106: 422, 134: 447, 415, 142: 448}, + // 215 + {65, 65, 65, 8: 65, 65, 11: 65, 13: 423, 129: 425, 138: 424}, + {66, 66, 66, 8: 66, 66, 11: 66, 13: 66}, + {114: 440}, + {63, 63, 63, 8: 63, 63, 11: 426, 133: 428, 141: 427}, + {64, 64, 64, 8: 64, 64, 11: 64}, + // 220 + {114: 435}, + {76, 76, 76, 8: 76, 430, 139: 429}, + {62, 62, 62, 8: 62, 62}, + {74, 74, 74, 8: 433, 140: 432}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 431}, + // 225 + {75, 75, 75, 8: 75, 14: 308, 307, 90: 306}, + {78, 78, 78}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 434}, + {73, 73, 73, 14: 308, 307, 90: 306}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 322, 94: 436}, + // 230 + {111, 111, 111, 8: 111, 111, 17: 438, 439, 170: 437}, + {112, 112, 112, 8: 112, 112}, + {110, 110, 110, 8: 110, 110}, + {109, 109, 109, 8: 109, 109}, + {10: 240, 91: 441, 115: 442}, + // 235 + {190, 190, 190, 190, 8: 190, 190, 11: 190, 155: 443}, + {133, 133, 133, 8: 133, 133, 11: 133}, + {188, 188, 188, 445, 8: 188, 188, 11: 188, 156: 444}, + {191, 191, 191, 8: 191, 191, 11: 191}, + {187, 187, 187, 8: 187, 187, 240, 187, 91: 446}, + // 240 + {189, 189, 189, 189, 8: 189, 189, 11: 189}, + {80, 80, 80, 80, 8: 80, 80, 11: 80, 80, 80}, + {65, 65, 65, 8: 65, 65, 11: 65, 13: 423, 129: 425, 138: 449}, + {63, 63, 63, 8: 63, 63, 11: 426, 133: 428, 141: 450}, + {76, 76, 76, 8: 76, 430, 139: 451}, + // 245 + {74, 74, 74, 8: 433, 140: 452}, + {77, 77, 77}, + {455, 2: 85, 172: 454}, + {2: 456}, + {2: 84}, + // 250 + {86, 86, 86, 86, 8: 86, 86, 11: 86, 86, 86, 20: 86}, + {88, 88, 88, 88, 8: 88, 88, 11: 88, 88, 88}, + {10: 459}, + {82, 82, 82, 82, 8: 82, 82, 11: 82, 82, 82}, + {4: 260, 295, 294, 292, 10: 266, 16: 68, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 409, 128: 461}, + // 255 + {3: 134, 16: 134}, + {3: 138, 16: 138}, + {10: 464}, + {3: 136, 16: 136}, + {10: 234, 93: 466}, + // 260 + {4: 468, 92: 129, 105: 129, 165: 467}, + {92: 216, 95: 472, 105: 471}, + {10: 240, 91: 441, 115: 469}, + {2: 470}, + {92: 128, 105: 128}, + // 265 + {4: 473}, + {130, 130}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 322, 94: 474}, + {2: 475}, + {127, 127, 3: 127, 166: 476}, + // 270 + {125, 125, 3: 478, 167: 477}, + {131, 131}, + {124, 124, 4: 479}, + {4: 260, 295, 294, 292, 10: 266, 26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 251, 284, 285, 286, 288, 289, 290, 291, 287, 56: 250, 253, 59: 254, 255, 65: 258, 256, 252, 293, 245, 71: 262, 257, 261, 263, 259, 82: 265, 264, 249, 86: 267, 248, 246, 322, 94: 480}, + {2: 481}, + // 275 + {126, 126, 3: 126}, + {10: 169, 102: 489, 160: 488}, + {10: 234, 93: 484, 102: 485}, + {167, 167}, + {101: 486}, + // 280 + {10: 234, 93: 487}, + {166, 166}, + {10: 491}, + {101: 490}, + {10: 168}, + // 285 + {170, 170}, + {10: 234, 93: 493}, + {172, 172, 12: 244, 106: 494}, + {171, 171}, + {103: 517}, + // 290 + {103: 179}, + {10: 234, 93: 498, 102: 499}, + {4: 512}, + {25: 500}, + {101: 501}, + // 295 + {10: 234, 93: 502}, + {4: 503}, + {10: 240, 91: 504, 99: 505}, + {26: 268, 269, 270, 271, 272, 273, 274, 275, 277, 278, 276, 280, 281, 282, 283, 279, 43: 284, 285, 286, 288, 289, 290, 291, 287, 69: 511}, + {2: 176, 176, 121: 506}, + // 300 + {2: 174, 508, 122: 507}, + {2: 510}, + {2: 173, 10: 240, 91: 504, 99: 509}, + {2: 175, 175}, + {177, 177}, + // 305 + {193, 193, 193, 193}, + {10: 240, 91: 504, 99: 513}, + {2: 176, 176, 121: 514}, + {2: 174, 508, 122: 515}, + {2: 516}, + // 310 + {178, 178}, + {10: 182, 102: 519, 157: 518}, + {10: 522}, + {25: 520}, + {101: 521}, + // 315 + {10: 181}, + {169: 523}, + {10: 524}, + {4: 525}, + {10: 526}, + // 320 + {2: 527, 4: 528}, + {184, 184}, + {2: 529}, + {2: 530}, + {183, 183}, + // 325 + {197, 197}, + {10: 234, 93: 533}, + {100: 535, 108: 534}, + {10: 240, 91: 504, 99: 538}, + {154: 536}, + // 330 + {10: 240, 91: 537}, + {204, 204}, + {205, 205}, + {165, 165, 92: 216, 95: 228, 100: 213, 109: 208, 218, 112: 209, 219, 116: 210, 220, 211, 221, 222, 123: 223, 212, 224, 225, 217, 130: 214, 226, 136: 215, 227, 144: 540, 232, 229, 233, 230}, + {42, 42}, + } +) + +var yyDebug = 0 + +type yyLexer interface { + Lex(lval *yySymType) int + Error(s string) +} + +type yyLexerEx interface { + yyLexer + Reduced(rule, state int, lval *yySymType) bool +} + +func yySymName(c int) (s string) { + x, ok := yyXLAT[c] + if ok { + return yySymNames[x] + } + + return __yyfmt__.Sprintf("%d", c) +} + +func yylex1(yylex yyLexer, lval *yySymType) (n int) { + n = yylex.Lex(lval) + if n <= 0 { + n = yyEOFCode + } + if yyDebug >= 3 { + __yyfmt__.Printf("\nlex %s(%#x %d), lval: %+v\n", yySymName(n), n, n, lval) + } + return n +} + +func yyParse(yylex yyLexer) int { + const yyError = 182 + + yyEx, _ := yylex.(yyLexerEx) + var yyn int + var yylval yySymType + var yyVAL yySymType + yyS := make([]yySymType, 200) + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + yyerrok := func() { + if yyDebug >= 2 { + __yyfmt__.Printf("yyerrok()\n") + } + Errflag = 0 + } + _ = yyerrok + yystate := 0 + yychar := -1 + var yyxchar int + yyp := -1 + goto yystack + +ret0: + return 0 + +ret1: + return 1 + +yystack: + /* put a state and value onto the stack */ + yyp++ + if yyp >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyS[yyp] = yyVAL + yyS[yyp].yys = yystate + +yynewstate: + if yychar < 0 { + yychar = yylex1(yylex, &yylval) + var ok bool + if yyxchar, ok = yyXLAT[yychar]; !ok { + yyxchar = len(yySymNames) // > tab width + } + } + if yyDebug >= 4 { + var a []int + for _, v := range yyS[:yyp+1] { + a = append(a, v.yys) + } + __yyfmt__.Printf("state stack %v\n", a) + } + row := yyParseTab[yystate] + yyn = 0 + if yyxchar < len(row) { + if yyn = int(row[yyxchar]); yyn != 0 { + yyn += yyTabOfs + } + } + switch { + case yyn > 0: // shift + yychar = -1 + yyVAL = yylval + yystate = yyn + if yyDebug >= 2 { + __yyfmt__.Printf("shift, and goto state %d\n", yystate) + } + if Errflag > 0 { + Errflag-- + } + goto yystack + case yyn < 0: // reduce + case yystate == 1: // accept + if yyDebug >= 2 { + __yyfmt__.Println("accept") + } + goto ret0 + } + + if yyn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + if yyDebug >= 1 { + __yyfmt__.Printf("no action for %s in state %d\n", yySymName(yychar), yystate) + } + k := yyXError{yystate, yyxchar} + msg, ok := yyXErrors[k] + if !ok { + k.xsym = -1 + msg, ok = yyXErrors[k] + } + if !ok { + msg = "syntax error" + } + if msg != "" { + yylex.Error(msg) + } + Nerrs++ + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for yyp >= 0 { + row := yyParseTab[yyS[yyp].yys] + if yyError < len(row) { + yyn = int(row[yyError]) + yyTabOfs + if yyn > 0 { // hit + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery found error shift in state %d\n", yyS[yyp].yys) + } + yystate = yyn /* simulate a shift of "error" */ + goto yystack + } + } + + /* the current p has no shift on "error", pop stack */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys) + } + yyp-- + } + /* there is no state on the stack with an error shift ... abort */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery failed\n") + } + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", yySymName(yychar)) + } + if yychar == yyEOFCode { + goto ret1 + } + + yychar = -1 + goto yynewstate /* try again in the same state */ + } + } + + r := -yyn + x0 := yyReductions[r] + x, n := x0.xsym, x0.components + yypt := yyp + _ = yypt // guard against "declared and not used" + + yyp -= n + if yyp+1 >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyVAL = yyS[yyp+1] + + /* consult goto table to find next state */ + exState := yystate + yystate = int(yyParseTab[yyS[yyp].yys][x]) + yyTabOfs + /* reduction by production r */ + if yyDebug >= 2 { + __yyfmt__.Printf("reduce using rule %v (%s), and goto state %d\n", r, yySymNames[x], yystate) + } + + switch r { + case 1: + { + yyVAL.item = &alterTableAddStmt{tableName: yyS[yypt-2].item.(string), c: yyS[yypt-0].item.(*col)} + } + case 2: + { + yyVAL.item = &alterTableDropColumnStmt{tableName: yyS[yypt-3].item.(string), colName: yyS[yypt-0].item.(string)} + } + case 3: + { + yyVAL.item = assignment{colName: yyS[yypt-2].item.(string), expr: yyS[yypt-0].item.(expression)} + } + case 4: + { + yyVAL.item = append([]assignment{yyS[yypt-2].item.(assignment)}, yyS[yypt-1].item.([]assignment)...) + } + case 5: + { + yyVAL.item = []assignment{} + } + case 6: + { + yyVAL.item = append(yyS[yypt-2].item.([]assignment), yyS[yypt-0].item.(assignment)) + } + case 9: + { + yyVAL.item = beginTransactionStmt{} + } + case 10: + { + yyVAL.item = yyS[yypt-1].item + } + case 11: + { + yyVAL.item = []expression{} + } + case 13: + { + yyVAL.item = &col{name: yyS[yypt-1].item.(string), typ: yyS[yypt-0].item.(int)} + } + case 15: + { + yyVAL.item = append([]string{yyS[yypt-2].item.(string)}, yyS[yypt-1].item.([]string)...) + } + case 16: + { + yyVAL.item = []string{} + } + case 17: + { + yyVAL.item = append(yyS[yypt-2].item.([]string), yyS[yypt-0].item.(string)) + } + case 20: + { + yyVAL.item = commitStmt{} + } + case 21: + { + yyVAL.item = &conversion{typ: yyS[yypt-3].item.(int), val: yyS[yypt-1].item.(expression)} + } + case 22: + { + indexName, tableName, columnName := yyS[yypt-5].item.(string), yyS[yypt-3].item.(string), yyS[yypt-1].item.(string) + yyVAL.item = &createIndexStmt{unique: yyS[yypt-8].item.(bool), ifNotExists: yyS[yypt-6].item.(bool), indexName: indexName, tableName: tableName, colName: columnName} + if indexName == tableName || indexName == columnName { + yylex.(*lexer).err("index name collision: %s", indexName) + return 1 + } + + if isSystemName[indexName] || isSystemName[tableName] { + yylex.(*lexer).err("name is used for system tables: %s", indexName) + return 1 + } + } + case 23: + { + indexName, tableName, columnName := yyS[yypt-7].item.(string), yyS[yypt-5].item.(string), yyS[yypt-3].item.(string) + yyVAL.item = &createIndexStmt{unique: yyS[yypt-10].item.(bool), ifNotExists: yyS[yypt-8].item.(bool), indexName: indexName, tableName: tableName, colName: "id()"} + if yyS[yypt-3].item.(string) != "id" { + yylex.(*lexer).err("only the built-in function id() can be used in index: %s()", columnName) + return 1 + } + + if indexName == tableName { + yylex.(*lexer).err("index name collision: %s", indexName) + return 1 + } + + if isSystemName[indexName] || isSystemName[tableName] { + yylex.(*lexer).err("name is used for system tables: %s", indexName) + return 1 + } + } + case 24: + { + yyVAL.item = false + } + case 25: + { + yyVAL.item = true + } + case 26: + { + yyVAL.item = false + } + case 27: + { + yyVAL.item = true + } + case 28: + { + nm := yyS[yypt-5].item.(string) + yyVAL.item = &createTableStmt{tableName: nm, cols: append([]*col{yyS[yypt-3].item.(*col)}, yyS[yypt-2].item.([]*col)...)} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } + case 29: + { + nm := yyS[yypt-5].item.(string) + yyVAL.item = &createTableStmt{ifNotExists: true, tableName: nm, cols: append([]*col{yyS[yypt-3].item.(*col)}, yyS[yypt-2].item.([]*col)...)} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } + case 30: + { + yyVAL.item = []*col{} + } + case 31: + { + yyVAL.item = append(yyS[yypt-2].item.([]*col), yyS[yypt-0].item.(*col)) + } + case 34: + { + yyVAL.item = &truncateTableStmt{yyS[yypt-0].item.(string)} + } + case 35: + { + yyVAL.item = &deleteStmt{tableName: yyS[yypt-1].item.(string), where: yyS[yypt-0].item.(*whereRset).expr} + } + case 36: + { + yyVAL.item = &dropIndexStmt{ifExists: yyS[yypt-1].item.(bool), indexName: yyS[yypt-0].item.(string)} + } + case 37: + { + yyVAL.item = false + } + case 38: + { + yyVAL.item = true + } + case 39: + { + nm := yyS[yypt-0].item.(string) + yyVAL.item = &dropTableStmt{tableName: nm} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } + case 40: + { + nm := yyS[yypt-0].item.(string) + yyVAL.item = &dropTableStmt{ifExists: true, tableName: nm} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } + case 41: + { + yyVAL.item = nil + } + case 43: + { + var err error + if yyVAL.item, err = newBinaryOperation(oror, yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 46: + { + yyVAL.item = append([]expression{yyS[yypt-2].item.(expression)}, yyS[yypt-1].item.([]expression)...) + } + case 47: + { + yyVAL.item = []expression(nil) + } + case 48: + { + yyVAL.item = append(yyS[yypt-2].item.([]expression), yyS[yypt-0].item.(expression)) + } + case 52: + { + yyVAL.item = &pIn{expr: yyS[yypt-4].item.(expression), list: yyS[yypt-1].item.([]expression)} + } + case 53: + { + yyVAL.item = &pIn{expr: yyS[yypt-5].item.(expression), not: true, list: yyS[yypt-1].item.([]expression)} + } + case 54: + { + yyVAL.item = &pIn{expr: yyS[yypt-4].item.(expression), sel: yyS[yypt-1].item.(*selectStmt)} + } + case 55: + { + yyVAL.item = &pIn{expr: yyS[yypt-5].item.(expression), not: true, sel: yyS[yypt-1].item.(*selectStmt)} + } + case 56: + { + var err error + if yyVAL.item, err = newBetween(yyS[yypt-4].item, yyS[yypt-2].item, yyS[yypt-0].item, false); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 57: + { + var err error + if yyVAL.item, err = newBetween(yyS[yypt-5].item, yyS[yypt-2].item, yyS[yypt-0].item, true); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 58: + { + yyVAL.item = &isNull{expr: yyS[yypt-2].item.(expression)} + } + case 59: + { + yyVAL.item = &isNull{expr: yyS[yypt-3].item.(expression), not: true} + } + case 61: + { + var err error + if yyVAL.item, err = newBinaryOperation(ge, yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 62: + { + var err error + if yyVAL.item, err = newBinaryOperation('>', yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 63: + { + var err error + if yyVAL.item, err = newBinaryOperation(le, yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 64: + { + var err error + if yyVAL.item, err = newBinaryOperation('<', yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 65: + { + var err error + if yyVAL.item, err = newBinaryOperation(neq, yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 66: + { + var err error + if yyVAL.item, err = newBinaryOperation(eq, yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 67: + { + yyVAL.item = &pLike{expr: yyS[yypt-2].item.(expression), pattern: yyS[yypt-0].item.(expression)} + } + case 68: + { + expr, name := yyS[yypt-1].item.(expression), yyS[yypt-0].item.(string) + if name == "" { + s, ok := expr.(*ident) + if ok { + name = s.s + } + } + yyVAL.item = &fld{expr: expr, name: name} + } + case 69: + { + yyVAL.item = "" + } + case 70: + { + yyVAL.item = yyS[yypt-0].item + } + case 71: + { + yyVAL.item = []*fld{yyS[yypt-0].item.(*fld)} + } + case 72: + { + l, f := yyS[yypt-2].item.([]*fld), yyS[yypt-0].item.(*fld) + if f.name != "" { + if f := findFld(l, f.name); f != nil { + yylex.(*lexer).err("duplicate field name %q", f.name) + return 1 + } + } + + yyVAL.item = append(yyS[yypt-2].item.([]*fld), yyS[yypt-0].item.(*fld)) + } + case 73: + { + yyVAL.item = &groupByRset{colNames: yyS[yypt-0].item.([]string)} + } + case 74: + { + yyVAL.item = yyS[yypt-1].item + } + case 75: + { + yyVAL.item = &insertIntoStmt{tableName: yyS[yypt-7].item.(string), colNames: yyS[yypt-6].item.([]string), lists: append([][]expression{yyS[yypt-3].item.([]expression)}, yyS[yypt-1].item.([][]expression)...)} + } + case 76: + { + yyVAL.item = &insertIntoStmt{tableName: yyS[yypt-2].item.(string), colNames: yyS[yypt-1].item.([]string), sel: yyS[yypt-0].item.(*selectStmt)} + } + case 77: + { + yyVAL.item = []string{} + } + case 78: + { + yyVAL.item = yyS[yypt-1].item + } + case 79: + { + yyVAL.item = [][]expression{} + } + case 80: + { + yyVAL.item = append(yyS[yypt-4].item.([][]expression), yyS[yypt-1].item.([]expression)) + } + case 90: + { + yyVAL.item = value{yyS[yypt-0].item} + } + case 91: + { + n := yyS[yypt-0].item.(int) + yyVAL.item = parameter{n} + l := yylex.(*lexer) + l.params = mathutil.Max(l.params, n) + if n == 0 { + l.err("parameter number must be non zero") + return 1 + } + } + case 92: + { + yyVAL.item = &ident{yyS[yypt-0].item.(string)} + } + case 93: + { + yyVAL.item = &pexpr{expr: yyS[yypt-1].item.(expression)} + } + case 94: + { + yyVAL.item = &orderByRset{by: yyS[yypt-1].item.([]expression), asc: yyS[yypt-0].item.(bool)} + } + case 95: + { + yyVAL.item = true // ASC by default + } + case 96: + { + yyVAL.item = true + } + case 97: + { + yyVAL.item = false + } + case 100: + { + var err error + if yyVAL.item, err = newIndex(yyS[yypt-1].item.(expression), yyS[yypt-0].item.(expression)); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 101: + { + var err error + s := yyS[yypt-0].item.([2]*expression) + if yyVAL.item, err = newSlice(yyS[yypt-1].item.(expression), s[0], s[1]); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 102: + { + x := yylex.(*lexer) + f, ok := yyS[yypt-1].item.(*ident) + if !ok { + x.err("expected identifier or qualified identifier") + return 1 + } + + var err error + var agg bool + if yyVAL.item, agg, err = newCall(f.s, yyS[yypt-0].item.([]expression)); err != nil { + x.err("%v", err) + return 1 + } + if n := len(x.agg); n > 0 { + x.agg[n-1] = x.agg[n-1] || agg + } + } + case 104: + { + var err error + if yyVAL.item, err = newBinaryOperation('^', yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 105: + { + var err error + if yyVAL.item, err = newBinaryOperation('|', yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 106: + { + var err error + if yyVAL.item, err = newBinaryOperation('-', yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 107: + { + var err error + yyVAL.item, err = newBinaryOperation('+', yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 109: + { + var err error + yyVAL.item, err = newBinaryOperation(andnot, yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 110: + { + var err error + yyVAL.item, err = newBinaryOperation('&', yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 111: + { + var err error + yyVAL.item, err = newBinaryOperation(lsh, yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 112: + { + var err error + yyVAL.item, err = newBinaryOperation(rsh, yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 113: + { + var err error + yyVAL.item, err = newBinaryOperation('%', yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 114: + { + var err error + yyVAL.item, err = newBinaryOperation('/', yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 115: + { + var err error + yyVAL.item, err = newBinaryOperation('*', yyS[yypt-2].item, yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 117: + { + yyVAL.item = fmt.Sprintf("%s.%s", yyS[yypt-2].item.(string), yyS[yypt-0].item.(string)) + } + case 118: + { + yyVAL.item = []interface{}{yyS[yypt-1].item, yyS[yypt-0].item} + } + case 120: + { + yyVAL.item = yyS[yypt-2].item + } + case 123: + { + yyVAL.item = "" + } + case 124: + { + yyVAL.item = yyS[yypt-0].item + } + case 125: + { + yyVAL.list = []interface{}{yyS[yypt-0].item} + } + case 126: + { + yyVAL.list = append(yyS[yypt-2].list, yyS[yypt-0].item) + } + case 127: + { + yyVAL.item = rollbackStmt{} + } + case 128: + { + x := yylex.(*lexer) + n := len(x.agg) + yyVAL.item = &selectStmt{ + distinct: yyS[yypt-8].item.(bool), + flds: yyS[yypt-7].item.([]*fld), + from: &crossJoinRset{sources: yyS[yypt-5].list}, + hasAggregates: x.agg[n-1], + where: yyS[yypt-4].item.(*whereRset), + group: yyS[yypt-3].item.(*groupByRset), + order: yyS[yypt-2].item.(*orderByRset), + limit: yyS[yypt-1].item.(*limitRset), + offset: yyS[yypt-0].item.(*offsetRset), + } + x.agg = x.agg[:n-1] + } + case 129: + { + x := yylex.(*lexer) + n := len(x.agg) + yyVAL.item = &selectStmt{ + distinct: yyS[yypt-9].item.(bool), + flds: yyS[yypt-8].item.([]*fld), + from: &crossJoinRset{sources: yyS[yypt-6].list}, + hasAggregates: x.agg[n-1], + where: yyS[yypt-4].item.(*whereRset), + group: yyS[yypt-3].item.(*groupByRset), + order: yyS[yypt-2].item.(*orderByRset), + limit: yyS[yypt-1].item.(*limitRset), + offset: yyS[yypt-0].item.(*offsetRset), + } + x.agg = x.agg[:n-1] + } + case 130: + { + yyVAL.item = (*limitRset)(nil) + } + case 131: + { + yyVAL.item = &limitRset{expr: yyS[yypt-0].item.(expression)} + } + case 132: + { + yyVAL.item = (*offsetRset)(nil) + } + case 133: + { + yyVAL.item = &offsetRset{expr: yyS[yypt-0].item.(expression)} + } + case 134: + { + yyVAL.item = false + } + case 135: + { + yyVAL.item = true + } + case 136: + { + yyVAL.item = []*fld{} + } + case 137: + { + yyVAL.item = yyS[yypt-0].item + } + case 138: + { + yyVAL.item = yyS[yypt-1].item + } + case 139: + { + yyVAL.item = (*whereRset)(nil) + } + case 141: + { + yyVAL.item = (*groupByRset)(nil) + } + case 143: + { + yyVAL.item = (*orderByRset)(nil) + } + case 145: + { + yyVAL.item = [2]*expression{nil, nil} + } + case 146: + { + hi := yyS[yypt-1].item.(expression) + yyVAL.item = [2]*expression{nil, &hi} + } + case 147: + { + lo := yyS[yypt-2].item.(expression) + yyVAL.item = [2]*expression{&lo, nil} + } + case 148: + { + lo := yyS[yypt-3].item.(expression) + hi := yyS[yypt-1].item.(expression) + yyVAL.item = [2]*expression{&lo, &hi} + } + case 163: + { + if yyS[yypt-0].item != nil { + yylex.(*lexer).list = []stmt{yyS[yypt-0].item.(stmt)} + } + } + case 164: + { + if yyS[yypt-0].item != nil { + yylex.(*lexer).list = append(yylex.(*lexer).list, yyS[yypt-0].item.(stmt)) + } + } + case 167: + { + var err error + if yyVAL.item, err = newBinaryOperation(andand, yyS[yypt-2].item, yyS[yypt-0].item); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 170: + { + yyVAL.item = &truncateTableStmt{tableName: yyS[yypt-0].item.(string)} + } + case 195: + { + yyVAL.item = &updateStmt{tableName: yyS[yypt-3].item.(string), list: yyS[yypt-1].item.([]assignment), where: yyS[yypt-0].item.(*whereRset).expr} + } + case 196: + { + yyVAL.item = nowhere + } + case 199: + { + var err error + yyVAL.item, err = newUnaryOperation('^', yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 200: + { + var err error + yyVAL.item, err = newUnaryOperation('!', yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 201: + { + var err error + yyVAL.item, err = newUnaryOperation('-', yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 202: + { + var err error + yyVAL.item, err = newUnaryOperation('+', yyS[yypt-0].item) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + case 203: + { + yyVAL.item = &whereRset{expr: yyS[yypt-0].item.(expression)} + } + + } + + if yyEx != nil && yyEx.Reduced(r, exState, &yyVAL) { + return -1 + } + goto yystack /* stack new state and value */ +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/parser.y b/Godeps/_workspace/src/github.com/cznic/ql/parser.y new file mode 100644 index 00000000000..eeb489accc4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/parser.y @@ -0,0 +1,1062 @@ +%{ + +// Copyright (c) 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Inital yacc source generated by ebnf2y[1] +// at 2013-10-04 23:10:47.861401015 +0200 CEST +// +// $ ebnf2y -o ql.y -oe ql.ebnf -start StatementList -pkg ql -p _ +// +// CAUTION: If this file is a Go source file (*.go), it was generated +// automatically by '$ goyacc' from a *.y file - DO NOT EDIT in that case! +// +// [1]: http://github.com/cznic/ebnf2y + +package ql + +import ( + "fmt" + + "github.com/cznic/mathutil" +) + +%} + +%union { + line int + col int + item interface{} + list []interface{} +} + +%token add alter and andand andnot as asc + begin between bigIntType bigRatType blobType boolType by byteType + column commit complex128Type complex64Type create + deleteKwd desc distinct drop durationType + eq exists + falseKwd floatType float32Type float64Type floatLit from + ge group + identifier ifKwd imaginaryLit in index insert intType int16Type + int32Type int64Type int8Type into intLit is + le like limit lsh + neq not null + offset on or order oror + qlParam + rollback rsh runeType + selectKwd set stringType stringLit + tableKwd timeType transaction trueKwd truncate + uintType uint16Type uint32Type uint64Type uint8Type unique update + values + where + +%token + floatLit imaginaryLit intLit stringLit + +%token + bigIntType bigRatType blobType boolType byteType + complex64Type complex128Type + durationType + falseKwd floatType float32Type float64Type + identifier intType int16Type int32Type int64Type int8Type + null + qlParam + runeType + stringType + timeType trueKwd + uintType uint16Type uint32Type uint64Type uint8Type + +%type + AlterTableStmt Assignment AssignmentList AssignmentList1 + BeginTransactionStmt + Call Call1 ColumnDef ColumnName ColumnNameList ColumnNameList1 + CommitStmt Conversion CreateIndexStmt CreateIndexIfNotExists + CreateIndexStmtUnique CreateTableStmt CreateTableStmt1 + DeleteFromStmt DropIndexStmt DropIndexIfExists DropTableStmt + EmptyStmt Expression ExpressionList ExpressionList1 + Factor Factor1 Field Field1 FieldList + GroupByClause + Index InsertIntoStmt InsertIntoStmt1 InsertIntoStmt2 + Literal + Operand OrderBy OrderBy1 + QualifiedIdent + PrimaryExpression PrimaryFactor PrimaryTerm + RecordSet RecordSet1 RecordSet2 RollbackStmt + SelectStmt SelectStmtDistinct SelectStmtFieldList SelectStmtLimit + SelectStmtWhere SelectStmtGroup SelectStmtOffset SelectStmtOrder Slice + Statement StatementList + TableName Term TruncateTableStmt Type + UnaryExpr UpdateStmt UpdateStmt1 + WhereClause + +%type RecordSetList + +%start StatementList + +%% + +AlterTableStmt: + alter tableKwd TableName add ColumnDef + { + $$ = &alterTableAddStmt{tableName: $3.(string), c: $5.(*col)} + } +| alter tableKwd TableName drop column ColumnName + { + $$ = &alterTableDropColumnStmt{tableName: $3.(string), colName: $6.(string)} + } + +Assignment: + ColumnName '=' Expression + { + $$ = assignment{colName: $1.(string), expr: $3.(expression)} + } + +AssignmentList: + Assignment AssignmentList1 AssignmentList2 + { + $$ = append([]assignment{$1.(assignment)}, $2.([]assignment)...) + } + +AssignmentList1: + /* EMPTY */ + { + $$ = []assignment{} + } +| AssignmentList1 ',' Assignment + { + $$ = append($1.([]assignment), $3.(assignment)) + } + +AssignmentList2: + /* EMPTY */ +| ',' + +BeginTransactionStmt: + begin transaction + { + $$ = beginTransactionStmt{} + } + +Call: + '(' Call1 ')' + { + $$ = $2 + } + +Call1: + /* EMPTY */ + { + $$ = []expression{} + } +| ExpressionList + +ColumnDef: + ColumnName Type + { + $$ = &col{name: $1.(string), typ: $2.(int)} + } + +ColumnName: + identifier + +ColumnNameList: + ColumnName ColumnNameList1 ColumnNameList2 + { + $$ = append([]string{$1.(string)}, $2.([]string)...) + } + +ColumnNameList1: + /* EMPTY */ + { + $$ = []string{} + } +| ColumnNameList1 ',' ColumnName + { + $$ = append($1.([]string), $3.(string)) + } + +ColumnNameList2: + /* EMPTY */ +| ',' + +CommitStmt: + commit + { + $$ = commitStmt{} + } + +Conversion: + Type '(' Expression ')' + { + $$ = &conversion{typ: $1.(int), val: $3.(expression)} + } + +CreateIndexStmt: + create CreateIndexStmtUnique index CreateIndexIfNotExists identifier on identifier '(' identifier ')' + { + indexName, tableName, columnName := $5.(string), $7.(string), $9.(string) + $$ = &createIndexStmt{unique: $2.(bool), ifNotExists: $4.(bool), indexName: indexName, tableName: tableName, colName: columnName} + if indexName == tableName || indexName == columnName { + yylex.(*lexer).err("index name collision: %s", indexName) + return 1 + } + + if isSystemName[indexName] || isSystemName[tableName] { + yylex.(*lexer).err("name is used for system tables: %s", indexName) + return 1 + } + } +| create CreateIndexStmtUnique index CreateIndexIfNotExists identifier on identifier '(' identifier '(' ')' ')' + { + indexName, tableName, columnName := $5.(string), $7.(string), $9.(string) + $$ = &createIndexStmt{unique: $2.(bool), ifNotExists: $4.(bool), indexName: indexName, tableName: tableName, colName: "id()"} + if $9.(string) != "id" { + yylex.(*lexer).err("only the built-in function id() can be used in index: %s()", columnName) + return 1 + } + + if indexName == tableName { + yylex.(*lexer).err("index name collision: %s", indexName) + return 1 + } + + if isSystemName[indexName] || isSystemName[tableName] { + yylex.(*lexer).err("name is used for system tables: %s", indexName) + return 1 + } + } + +CreateIndexIfNotExists: + { + $$ = false + } +| ifKwd not exists + { + $$ = true + } + +CreateIndexStmtUnique: + { + $$ = false + } +| unique + { + $$ = true + } + +CreateTableStmt: + create tableKwd TableName '(' ColumnDef CreateTableStmt1 CreateTableStmt2 ')' + { + nm := $3.(string) + $$ = &createTableStmt{tableName: nm, cols: append([]*col{$5.(*col)}, $6.([]*col)...)} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } +| create tableKwd ifKwd not exists TableName '(' ColumnDef CreateTableStmt1 CreateTableStmt2 ')' + { + nm := $6.(string) + $$ = &createTableStmt{ifNotExists: true, tableName: nm, cols: append([]*col{$8.(*col)}, $9.([]*col)...)} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } + +CreateTableStmt1: + /* EMPTY */ + { + $$ = []*col{} + } +| CreateTableStmt1 ',' ColumnDef + { + $$ = append($1.([]*col), $3.(*col)) + } + +CreateTableStmt2: + /* EMPTY */ +| ',' + +DeleteFromStmt: + deleteKwd from TableName + { + $$ = &truncateTableStmt{$3.(string)} + } +| deleteKwd from TableName WhereClause + { + $$ = &deleteStmt{tableName: $3.(string), where: $4.(*whereRset).expr} + } + +DropIndexStmt: + drop index DropIndexIfExists identifier + { + $$ = &dropIndexStmt{ifExists: $3.(bool), indexName: $4.(string)} + } + +DropIndexIfExists: + { + $$ = false + } +| ifKwd exists + { + $$ = true + } + +DropTableStmt: + drop tableKwd TableName + { + nm := $3.(string) + $$ = &dropTableStmt{tableName: nm} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } +| drop tableKwd ifKwd exists TableName + { + nm := $5.(string) + $$ = &dropTableStmt{ifExists: true, tableName: nm} + if isSystemName[nm] { + yylex.(*lexer).err("name is used for system tables: %s", nm) + return 1 + } + } + +EmptyStmt: + /* EMPTY */ + { + $$ = nil + } + +Expression: + Term +| Expression logOr Term + { + var err error + if $$, err = newBinaryOperation(oror, $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + +logOr: + oror +| or + +ExpressionList: + Expression ExpressionList1 ExpressionList2 + { + $$ = append([]expression{$1.(expression)}, $2.([]expression)...) + } + +ExpressionList1: + /* EMPTY */ + { + $$ = []expression(nil) + } +| ExpressionList1 ',' Expression + { + $$ = append($1.([]expression), $3.(expression)) + } + +ExpressionList2: + /* EMPTY */ +| ',' + +Factor: + Factor1 +| Factor1 in '(' ExpressionList ')' + { + $$ = &pIn{expr: $1.(expression), list: $4.([]expression)} + } +| Factor1 not in '(' ExpressionList ')' + { + $$ = &pIn{expr: $1.(expression), not: true, list: $5.([]expression)} + } +| Factor1 in '(' SelectStmt ')' + { + $$ = &pIn{expr: $1.(expression), sel: $4.(*selectStmt)} + } +| Factor1 not in '(' SelectStmt ')' + { + $$ = &pIn{expr: $1.(expression), not: true, sel: $5.(*selectStmt)} + } +| Factor1 between PrimaryFactor and PrimaryFactor + { + var err error + if $$, err = newBetween($1, $3, $5, false); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 not between PrimaryFactor and PrimaryFactor + { + var err error + if $$, err = newBetween($1, $4, $6, true); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 is null + { + $$ = &isNull{expr: $1.(expression)} + } +| Factor1 is not null + { + $$ = &isNull{expr: $1.(expression), not: true} + } + +Factor1: + PrimaryFactor +| Factor1 ge PrimaryFactor + { + var err error + if $$, err = newBinaryOperation(ge, $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 '>' PrimaryFactor + { + var err error + if $$, err = newBinaryOperation('>', $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 le PrimaryFactor + { + var err error + if $$, err = newBinaryOperation(le, $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 '<' PrimaryFactor + { + var err error + if $$, err = newBinaryOperation('<', $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 neq PrimaryFactor + { + var err error + if $$, err = newBinaryOperation(neq, $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 eq PrimaryFactor + { + var err error + if $$, err = newBinaryOperation(eq, $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| Factor1 like PrimaryFactor + { + $$ = &pLike{expr: $1.(expression), pattern: $3.(expression)} + } + +Field: + Expression Field1 + { + expr, name := $1.(expression), $2.(string) + if name == "" { + s, ok := expr.(*ident) + if ok { + name = s.s + } + } + $$ = &fld{expr: expr, name: name} + } + +Field1: + /* EMPTY */ + { + $$ = "" + } +| as identifier + { + $$ = $2 + } + +FieldList: + Field + { + $$ = []*fld{$1.(*fld)} + } +| FieldList ',' Field + { + l, f := $1.([]*fld), $3.(*fld) + if f.name != "" { + if f := findFld(l, f.name); f != nil { + yylex.(*lexer).err("duplicate field name %q", f.name) + return 1 + } + } + + $$ = append($1.([]*fld), $3.(*fld)) + } + +GroupByClause: + group by ColumnNameList + { + $$ = &groupByRset{colNames: $3.([]string)} + } + +Index: + '[' Expression ']' + { + $$ = $2 + } + +InsertIntoStmt: + insert into TableName InsertIntoStmt1 values '(' ExpressionList ')' InsertIntoStmt2 InsertIntoStmt3 + { + $$ = &insertIntoStmt{tableName: $3.(string), colNames: $4.([]string), lists: append([][]expression{$7.([]expression)}, $9.([][]expression)...)} + } +| insert into TableName InsertIntoStmt1 SelectStmt + { + $$ = &insertIntoStmt{tableName: $3.(string), colNames: $4.([]string), sel: $5.(*selectStmt)} + } + +InsertIntoStmt1: + /* EMPTY */ + { + $$ = []string{} + } +| '(' ColumnNameList ')' + { + $$ = $2 + } + +InsertIntoStmt2: + /* EMPTY */ + { + $$ = [][]expression{} + } +| InsertIntoStmt2 ',' '(' ExpressionList ')' + { + $$ = append($1.([][]expression), $4.([]expression)) + } + +InsertIntoStmt3: +| ',' + + +Literal: + falseKwd +| null +| trueKwd +| floatLit +| imaginaryLit +| intLit +| stringLit + +Operand: + Literal + { + $$ = value{$1} + } +| qlParam + { + n := $1.(int) + $$ = parameter{n} + l := yylex.(*lexer) + l.params = mathutil.Max(l.params, n) + if n == 0 { + l.err("parameter number must be non zero") + return 1 + } + } +| QualifiedIdent + { + $$ = &ident{$1.(string)} + } +| '(' Expression ')' + { + $$ = &pexpr{expr: $2.(expression)} + } + +OrderBy: + order by ExpressionList OrderBy1 + { + $$ = &orderByRset{by: $3.([]expression), asc: $4.(bool)} + } + +OrderBy1: + /* EMPTY */ + { + $$ = true // ASC by default + } +| asc + { + $$ = true + } +| desc + { + $$ = false + } + +PrimaryExpression: + Operand +| Conversion +| PrimaryExpression Index + { + var err error + if $$, err = newIndex($1.(expression), $2.(expression)); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryExpression Slice + { + var err error + s := $2.([2]*expression) + if $$, err = newSlice($1.(expression), s[0], s[1]); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryExpression Call + { + x := yylex.(*lexer) + f, ok := $1.(*ident) + if !ok { + x.err("expected identifier or qualified identifier") + return 1 + } + + var err error + var agg bool + if $$, agg, err = newCall(f.s, $2.([]expression)); err != nil { + x.err("%v", err) + return 1 + } + if n := len(x.agg); n > 0 { + x.agg[n-1] = x.agg[n-1] || agg + } + } + +PrimaryFactor: + PrimaryTerm +| PrimaryFactor '^' PrimaryTerm + { + var err error + if $$, err = newBinaryOperation('^', $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryFactor '|' PrimaryTerm + { + var err error + if $$, err = newBinaryOperation('|', $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryFactor '-' PrimaryTerm + { + var err error + if $$, err = newBinaryOperation('-', $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryFactor '+' PrimaryTerm + { + var err error + $$, err = newBinaryOperation('+', $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + +PrimaryTerm: + UnaryExpr +| PrimaryTerm andnot UnaryExpr + { + var err error + $$, err = newBinaryOperation(andnot, $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryTerm '&' UnaryExpr + { + var err error + $$, err = newBinaryOperation('&', $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryTerm lsh UnaryExpr + { + var err error + $$, err = newBinaryOperation(lsh, $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryTerm rsh UnaryExpr + { + var err error + $$, err = newBinaryOperation(rsh, $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryTerm '%' UnaryExpr + { + var err error + $$, err = newBinaryOperation('%', $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryTerm '/' UnaryExpr + { + var err error + $$, err = newBinaryOperation('/', $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| PrimaryTerm '*' UnaryExpr + { + var err error + $$, err = newBinaryOperation('*', $1, $3) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + +QualifiedIdent: + identifier +| identifier '.' identifier + { + $$ = fmt.Sprintf("%s.%s", $1.(string), $3.(string)) + } + +RecordSet: + RecordSet1 RecordSet2 + { + $$ = []interface{}{$1, $2} + } + +RecordSet1: + identifier +| '(' SelectStmt RecordSet11 ')' + { + $$ = $2 + } + +RecordSet11: + /* EMPTY */ +| ';' + +RecordSet2: + /* EMPTY */ + { + $$ = "" + } +| as identifier + { + $$ = $2 + } + +RecordSetList: + RecordSet + { + $$ = []interface{}{$1} + } +| RecordSetList ',' RecordSet + { + $$ = append($1, $3) + } + +RollbackStmt: + rollback + { + $$ = rollbackStmt{} + } + +SelectStmt: + selectKwd SelectStmtDistinct SelectStmtFieldList from RecordSetList + SelectStmtWhere SelectStmtGroup SelectStmtOrder SelectStmtLimit SelectStmtOffset + { + x := yylex.(*lexer) + n := len(x.agg) + $$ = &selectStmt{ + distinct: $2.(bool), + flds: $3.([]*fld), + from: &crossJoinRset{sources: $5}, + hasAggregates: x.agg[n-1], + where: $6.(*whereRset), + group: $7.(*groupByRset), + order: $8.(*orderByRset), + limit: $9.(*limitRset), + offset: $10.(*offsetRset), + } + x.agg = x.agg[:n-1] + } +| selectKwd SelectStmtDistinct SelectStmtFieldList from RecordSetList ',' + SelectStmtWhere SelectStmtGroup SelectStmtOrder SelectStmtLimit SelectStmtOffset + { + x := yylex.(*lexer) + n := len(x.agg) + $$ = &selectStmt{ + distinct: $2.(bool), + flds: $3.([]*fld), + from: &crossJoinRset{sources: $5}, + hasAggregates: x.agg[n-1], + where: $7.(*whereRset), + group: $8.(*groupByRset), + order: $9.(*orderByRset), + limit: $10.(*limitRset), + offset: $11.(*offsetRset), + } + x.agg = x.agg[:n-1] + } + +SelectStmtLimit: + { + $$ = (*limitRset)(nil) + } +| limit Expression + { + $$ = &limitRset{expr: $2.(expression)} + } + +SelectStmtOffset: + { + $$ = (*offsetRset)(nil) + } +| offset Expression + { + $$ = &offsetRset{expr: $2.(expression)} + } + +SelectStmtDistinct: + /* EMPTY */ + { + $$ = false + } +| distinct + { + $$ = true + } + +SelectStmtFieldList: + '*' + { + $$ = []*fld{} + } +| FieldList + { + $$ = $1 + } +| FieldList ',' + { + $$ = $1 + } + +SelectStmtWhere: + /* EMPTY */ + { + $$ = (*whereRset)(nil) + } +| WhereClause + +SelectStmtGroup: + /* EMPTY */ + { + $$ = (*groupByRset)(nil) + } +| GroupByClause + +SelectStmtOrder: + /* EMPTY */ + { + $$ = (*orderByRset)(nil) + } +| OrderBy + +Slice: + '[' ':' ']' + { + $$ = [2]*expression{nil, nil} + } +| '[' ':' Expression ']' + { + hi := $3.(expression) + $$ = [2]*expression{nil, &hi} + } +| '[' Expression ':' ']' + { + lo := $2.(expression) + $$ = [2]*expression{&lo, nil} + } +| '[' Expression ':' Expression ']' + { + lo := $2.(expression) + hi := $4.(expression) + $$ = [2]*expression{&lo, &hi} + } + +Statement: + EmptyStmt +| AlterTableStmt +| BeginTransactionStmt +| CommitStmt +| CreateIndexStmt +| CreateTableStmt +| DeleteFromStmt +| DropIndexStmt +| DropTableStmt +| InsertIntoStmt +| RollbackStmt +| SelectStmt +| TruncateTableStmt +| UpdateStmt + +StatementList: + Statement + { + if $1 != nil { + yylex.(*lexer).list = []stmt{$1.(stmt)} + } + } +| StatementList ';' Statement + { + if $3 != nil { + yylex.(*lexer).list = append(yylex.(*lexer).list, $3.(stmt)) + } + } + +TableName: + identifier + +Term: + Factor +| Term logAnd Factor + { + var err error + if $$, err = newBinaryOperation(andand, $1, $3); err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + +logAnd: + andand +| and + +TruncateTableStmt: + truncate tableKwd TableName + { + $$ = &truncateTableStmt{tableName: $3.(string)} + } + +Type: + bigIntType +| bigRatType +| blobType +| boolType +| byteType +| complex128Type +| complex64Type +| durationType +| floatType +| float32Type +| float64Type +| intType +| int16Type +| int32Type +| int64Type +| int8Type +| runeType +| stringType +| timeType +| uintType +| uint16Type +| uint32Type +| uint64Type +| uint8Type + +UpdateStmt: + update TableName oSet AssignmentList UpdateStmt1 + { + $$ = &updateStmt{tableName: $2.(string), list: $4.([]assignment), where: $5.(*whereRset).expr} + } + +UpdateStmt1: + /* EMPTY */ + { + $$ = nowhere + } +| WhereClause + +UnaryExpr: + PrimaryExpression +| '^' PrimaryExpression + { + var err error + $$, err = newUnaryOperation('^', $2) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| '!' PrimaryExpression + { + var err error + $$, err = newUnaryOperation('!', $2) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| '-' PrimaryExpression + { + var err error + $$, err = newUnaryOperation('-', $2) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } +| '+' PrimaryExpression + { + var err error + $$, err = newUnaryOperation('+', $2) + if err != nil { + yylex.(*lexer).err("%v", err) + return 1 + } + } + +WhereClause: + where Expression + { + $$ = &whereRset{expr: $2.(expression)} + } + + +oSet: +| set diff --git a/Godeps/_workspace/src/github.com/cznic/ql/parser_test.go b/Godeps/_workspace/src/github.com/cznic/ql/parser_test.go new file mode 100644 index 00000000000..d603a2839fd --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/parser_test.go @@ -0,0 +1,193 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "testing" +) + +func TestParser0(t *testing.T) { + table := []struct { + src string + ok bool + }{ + {"", true}, + {";", true}, + {"CREATE", false}, + {"CREATE TABLE", false}, + {"CREATE TABLE foo (", false}, + // 5 + {"CREATE TABLE foo ()", false}, + {"CREATE TABLE foo ();", false}, + {"CREATE TABLE foo (a byte)", true}, + {"CREATE TABLE foo (a uint8);", true}, + {"CREATE TABLE foo (a uint16, b uint32)", true}, + // 10 + {"CREATE TABLE foo (a uint64, b bool);", true}, + {"CREATE TABLE foo (a int8, b int16) CREATE TABLE bar (x int32, y int64)", false}, + {"CREATE TABLE foo (a int, b float32); CREATE TABLE bar (x float64, y float)", true}, + {"INSERT INTO foo VALUES (1234)", true}, + {"INSERT INTO foo VALUES (1234, 5678)", true}, + // 15 + {"INSERT INTO foo VALUES (1 || 2)", false}, + {"INSERT INTO foo VALUES (1 | 2)", true}, + {"INSERT INTO foo VALUES (false || true)", true}, + {"INSERT INTO foo VALUES (id())", true}, + {"INSERT INTO foo VALUES (bar(5678))", false}, + // 20 + {"INSERT INTO foo VALUES ()", false}, + {"CREATE TABLE foo (a.b, b);", false}, + {"CREATE TABLE foo (a, b.c);", false}, + {"SELECT * FROM t", true}, + {"SELECT * FROM t AS u", true}, + // 25 + {"SELECT * FROM t, v", true}, + {"SELECT * FROM t AS u, v", true}, + {"SELECT * FROM t, v AS w", true}, + {"SELECT * FROM t AS u, v AS w", true}, + {"SELECT * FROM foo, bar, foo", true}, + // 30 + {"CREATE TABLE foo (a bytes)", false}, + {"SELECT DISTINCTS * FROM t", false}, + {"SELECT DISTINCT * FROM t", true}, + {"INSERT INTO foo (a) VALUES (42)", true}, + {"INSERT INTO foo (a,) VALUES (42,)", true}, + // 35 + {"INSERT INTO foo (a,b) VALUES (42,314)", true}, + {"INSERT INTO foo (a,b,) VALUES (42,314)", true}, + {"INSERT INTO foo (a,b,) VALUES (42,314,)", true}, + {"CREATE TABLE foo (a uint16, b uint32,)", true}, + {"CREATE TABLE foo (a uint16, b uint32,) -- foo", true}, + // 40 + {"CREATE TABLE foo (a uint16, b uint32,) // foo", true}, + {"CREATE TABLE foo (a uint16, b uint32,) /* foo */", true}, + {"CREATE TABLE foo /* foo */ (a uint16, b uint32,) /* foo */", true}, + {`-- Examples + ALTER TABLE Stock ADD Qty int; + + ALTER TABLE Income DROP COLUMN Taxes; + + CREATE TABLE department + ( + DepartmentID int, + DepartmentName string, // optional comma + ); + + CREATE TABLE employee + ( + LastName string, + DepartmentID int // optional comma + ); + + DROP TABLE Inventory; + + INSERT INTO department (DepartmentID) VALUES (42); + + INSERT INTO department ( + DepartmentName, + DepartmentID, + ) + VALUES ( + "R&D", + 42, + ); + + INSERT INTO department VALUES ( + 42, + "R&D", + ); + + SELECT * FROM Stock; + + SELECT DepartmentID + FROM department + WHERE DepartmentID == 42 + ORDER BY DepartmentName; + + SELECT employee.LastName + FROM department, employee + WHERE department.DepartmentID == employee.DepartmentID + ORDER BY DepartmentID; + + SELECT a.b, c.d + FROM + x AS a, + ( + SELECT * FROM y; // optional semicolon + ) AS c + WHERE a.e > c.e; + + SELECT a.b, c.d + FROM + x AS a, + ( + SELECT * FROM y // no semicolon + ) AS c + WHERE a.e > c.e; + + TRUNCATE TABLE department; + + SELECT DepartmentID + FROM department + WHERE DepartmentID == ?1 + ORDER BY DepartmentName; + + SELECT employee.LastName + FROM department, employee + WHERE department.DepartmentID == $1 && employee.LastName > $2 + ORDER BY DepartmentID; + + `, true}, + {"BEGIN TRANSACTION", true}, + // 45 + {"COMMIT", true}, + {"ROLLBACK", true}, + {` + BEGIN TRANSACTION; + INSERT INTO foo VALUES (42, 3.14); + INSERT INTO foo VALUES (-1, 2.78); + COMMIT;`, true}, + {` + BEGIN TRANSACTION; + INSERT INTO AccountA (Amount) VALUES ($1); + INSERT INTO AccountB (Amount) VALUES (-$1); + COMMIT;`, true}, + {` // A + BEGIN TRANSACTION; + INSERT INTO tmp SELECT * from bar; + SELECT * from tmp; + + // B + ROLLBACK;`, true}, + // 50 + {`-- 6 + ALTER TABLE none DROP COLUMN c1; + `, true}, + } + + for i, test := range table { + //dbg("%d ----\n%q\n----\n", i, test.src) + l := newLexer(test.src) + ok := yyParse(l) == 0 + if g, e := ok, test.ok; g != e { + if !ok { + t.Log(l.errs[0]) + } + t.Error(i, test.src, g, e) + return + } + + switch ok { + case true: + if len(l.errs) != 0 { + t.Fatal(l.errs) + } + case false: + if len(l.errs) == 0 { + t.Fatal(l.errs) + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/ql.ebnf b/Godeps/_workspace/src/github.com/cznic/ql/ql.ebnf new file mode 100644 index 00000000000..3c2876c4827 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/ql.ebnf @@ -0,0 +1,218 @@ +andand = "&&" . +andnot = "&^" . +ascii_letter = "a" … "z" | "A" … "Z" . +big_u_value = "\\" "U" hex_digit hex_digit hex_digit hex_digit hex_digit hex_digit hex_digit hex_digit . +byte_value = octal_byte_value | hex_byte_value . +decimal_digit = "0" … "9" . +decimal_lit = ( "1" … "9" ) { decimal_digit } . +decimals = decimal_digit { decimal_digit } . +eq = "==" . +escaped_char = "\\" ( + "a" + | "b" + | "f" + | "n" + | "r" + | "t" + | "v" + | "\\" + | "'" + | "\"" + ) . +exponent = ( "e" | "E" ) [ "+" | "-" ] decimals . +float_lit = decimals "." [ decimals ] [ exponent ] + | decimals exponent + | "." decimals [ exponent ] . +ge = ">=" . +hex_byte_value = "\\" "x" hex_digit hex_digit . +hex_digit = "0" … "9" + | "A" … "F" + | "a" … "f" . +hex_lit = "0" ( "x" | "X" ) hex_digit { hex_digit } . +identifier = letter { letter | decimal_digit } . +imaginary_lit = ( decimals | float_lit ) "i" . +int_lit = decimal_lit + | octal_lit + | hex_lit . +interpreted_string_lit = "\"" { unicode_value | byte_value } "\"" . +le = "<=" . +letter = ascii_letter | "_" . +little_u_value = "\\" "u" hex_digit hex_digit hex_digit hex_digit . +lsh = "<<" . +neq = "!=" . +newline = . +octal_byte_value = "\\" octal_digit octal_digit octal_digit . +octal_digit = "0" … "7" . +octal_lit = "0" { octal_digit } . +oror = "||" . +ql_parameter = ( "?" | "$" ) "1" … "9" { "0" … "9" } . +raw_string_lit = "`" { unicode_char | newline } "`" . +rsh = ">>" . +rune_lit = "'" ( unicode_value | byte_value ) "'" . +string_lit = raw_string_lit | interpreted_string_lit . +unicode_char = . +unicode_value = unicode_char + | little_u_value + | big_u_value + | escaped_char . + +AlterTableStmt = "ALTER" "TABLE" TableName ( + "ADD" ColumnDef + | "DROP" "COLUMN" ColumnName + ) . +Assignment = ColumnName "=" Expression . +AssignmentList = Assignment { "," Assignment } [ "," ] . +BeginTransactionStmt = "BEGIN" "TRANSACTION" . +Call = "(" [ ExpressionList ] ")" . +ColumnDef = ColumnName Type . +ColumnName = identifier . +ColumnNameList = ColumnName { "," ColumnName } [ "," ] . +CommitStmt = "COMMIT" . +Conversion = Type "(" Expression ")" . +CreateIndexStmt = "CREATE" [ "UNIQUE" ] "INDEX" [ + "IF" "NOT" "EXISTS" + ] IndexName "ON" TableName "(" ( + ColumnName + | "id" Call + ) ")" . +CreateTableStmt = "CREATE" "TABLE" [ + "IF" "NOT" "EXISTS" + ] TableName "(" ColumnDef { "," ColumnDef } [ "," ] ")" . +DeleteFromStmt = "DELETE" "FROM" TableName [ WhereClause ] . +DropIndexStmt = "DROP" "INDEX" [ "IF" "EXISTS" ] IndexName . +DropTableStmt = "DROP" "TABLE" [ "IF" "EXISTS" ] TableName . +EmptyStmt = . +Expression = Term { + ( oror | "OR" ) Term + } . +ExpressionList = Expression { "," Expression } [ "," ] . +Factor = PrimaryFactor { + ( + ge + | ">" + | le + | "<" + | neq + | eq + | "LIKE" + ) PrimaryFactor + } [ Predicate ] . +Field = Expression [ "AS" identifier ] . +FieldList = Field { "," Field } [ "," ] . +GroupByClause = "GROUP BY" ColumnNameList . +Index = "[" Expression "]" . +IndexName = identifier . +InsertIntoStmt = "INSERT" "INTO" TableName [ + "(" ColumnNameList ")" + ] ( Values | SelectStmt ) . +Limit = "Limit" Expression . +Literal = "FALSE" + | "NULL" + | "TRUE" + | float_lit + | imaginary_lit + | int_lit + | rune_lit + | string_lit + | ql_parameter . +Offset = "OFFSET" Expression . +Operand = Literal + | QualifiedIdent + | "(" Expression ")" . +OrderBy = "ORDER" "BY" ExpressionList [ "ASC" | "DESC" ] . +Predicate = ( + [ "NOT" ] ( + "IN" "(" ExpressionList ")" + | "IN" "(" SelectStmt ")" + | "BETWEEN" PrimaryFactor "AND" PrimaryFactor + ) + | "IS" [ "NOT" ] "NULL" + ) . +PrimaryExpression = Operand + | Conversion + | PrimaryExpression Index + | PrimaryExpression Slice + | PrimaryExpression Call . +PrimaryFactor = PrimaryTerm { + ( + "^" + | "|" + | "-" + | "+" + ) PrimaryTerm + } . +PrimaryTerm = UnaryExpr { + ( + andnot + | "&" + | lsh + | rsh + | "%" + | "/" + | "*" + ) UnaryExpr + } . +QualifiedIdent = identifier [ "." identifier ] . +RecordSet = ( + TableName + | "(" SelectStmt [ ";" ] ")" + ) [ "AS" identifier ] . +RecordSetList = RecordSet { "," RecordSet } [ "," ] . +RollbackStmt = "ROLLBACK" . +SelectStmt = "SELECT" [ "DISTINCT" ] ( "*" | FieldList ) "FROM" RecordSetList [ WhereClause ] [ GroupByClause ] [ OrderBy ] [ Limit ] [ Offset ] . +Slice = "[" [ Expression ] ":" [ Expression ] "]" . +Statement = EmptyStmt + | AlterTableStmt + | BeginTransactionStmt + | CommitStmt + | CreateIndexStmt + | CreateTableStmt + | DeleteFromStmt + | DropIndexStmt + | DropTableStmt + | InsertIntoStmt + | RollbackStmt + | SelectStmt + | TruncateTableStmt + | UpdateStmt . +StatementList = Statement { ";" Statement } . +TableName = identifier . +Term = Factor { + ( andand | "AND" ) Factor + } . +TruncateTableStmt = "TRUNCATE" "TABLE" TableName . +Type = "bigint" + | "bigrat" + | "blob" + | "bool" + | "byte" + | "complex128" + | "complex64" + | "duration" + | "float" + | "float32" + | "float64" + | "int" + | "int16" + | "int32" + | "int64" + | "int8" + | "rune" + | "string" + | "time" + | "uint" + | "uint16" + | "uint32" + | "uint64" + | "uint8" . +UnaryExpr = [ + "^" + | "!" + | "-" + | "+" + ] PrimaryExpression . +UpdateStmt = "UPDATE" TableName [ "SET" ] AssignmentList [ WhereClause ] . +Values = "VALUES" "(" ExpressionList ")" { + "," "(" ExpressionList ")" + } [ "," ] . +WhereClause = "WHERE" Expression . diff --git a/Godeps/_workspace/src/github.com/cznic/ql/ql.go b/Godeps/_workspace/src/github.com/cznic/ql/ql.go new file mode 100644 index 00000000000..d59e134e1a8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/ql.go @@ -0,0 +1,2162 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//LATER profile mem +//LATER profile cpu +//LATER coverage + +//MAYBE CROSSJOIN (explicit form), LEFT JOIN, INNER JOIN, OUTER JOIN equivalents. + +package ql + +import ( + "bytes" + "errors" + "fmt" + "io" + "log" + "math/big" + "strconv" + "strings" + "sync" + "time" + + "github.com/cznic/strutil" +) + +// NOTE: all rset implementations must be safe for concurrent use by multiple +// goroutines. If the do method requires any execution domain local data, they +// must be held out of the implementing instance. +var ( + _ rset = (*crossJoinRset)(nil) + _ rset = (*distinctRset)(nil) + _ rset = (*groupByRset)(nil) + _ rset = (*limitRset)(nil) + _ rset = (*offsetRset)(nil) + _ rset = (*orderByRset)(nil) + _ rset = (*selectRset)(nil) + _ rset = (*selectStmt)(nil) + _ rset = (*tableRset)(nil) + _ rset = (*whereRset)(nil) + + isTesting bool // enables test hook: select from an index +) + +const ( + noNames = iota + returnNames + onlyNames +) + +// List represents a group of compiled statements. +type List struct { + l []stmt + params int +} + +// String implements fmt.Stringer +func (l List) String() string { + var b bytes.Buffer + f := strutil.IndentFormatter(&b, "\t") + for _, s := range l.l { + switch s.(type) { + case beginTransactionStmt: + f.Format("%s\n%i", s) + case commitStmt, rollbackStmt: + f.Format("%u%s\n", s) + default: + f.Format("%s\n", s) + } + } + return b.String() +} + +type rset interface { + do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) error +} + +type recordset struct { + ctx *execCtx + rset + tx *TCtx +} + +func (r recordset) Do(names bool, f func(data []interface{}) (more bool, err error)) (err error) { + nm := noNames + if names { + nm = returnNames + } + return r.ctx.db.do(r, nm, f) +} + +func (r recordset) Fields() (names []string, err error) { + err = r.ctx.db.do( + r, + onlyNames, + func(data []interface{}) (more bool, err error) { + for _, v := range data { + s, ok := v.(string) + if !ok { + return false, fmt.Errorf("got %T(%v), expected string (RecordSet.Fields)", v, v) + } + names = append(names, s) + } + return false, nil + }, + ) + return +} + +func (r recordset) FirstRow() (row []interface{}, err error) { + rows, err := r.Rows(1, 0) + if err != nil { + return nil, err + } + + if len(rows) != 0 { + return rows[0], nil + } + + return nil, nil +} + +func (r recordset) Rows(limit, offset int) (rows [][]interface{}, err error) { + if err := r.Do(false, func(row []interface{}) (bool, error) { + if offset > 0 { + offset-- + return true, nil + } + + switch { + case limit < 0: + rows = append(rows, row) + return true, nil + case limit == 0: + return false, nil + default: // limit > 0 + rows = append(rows, row) + limit-- + return limit > 0, nil + } + }); err != nil { + return nil, err + } + + return rows, nil +} + +type groupByRset struct { + colNames []string + src rset +} + +func (r *groupByRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + t, err := ctx.db.store.CreateTemp(true) + if err != nil { + return + } + + defer func() { + if derr := t.Drop(); derr != nil && err == nil { + err = derr + } + }() + + var flds []*fld + var gcols []*col + var cols []*col + ok := false + k := make([]interface{}, len(r.colNames)) //LATER optimize when len(r.cols) == 0 + if err = r.src.do(ctx, onlyNames, func(rid interface{}, in []interface{}) (more bool, err error) { + if ok { + infer(in, &cols) + for i, c := range gcols { + k[i] = in[c.index] + } + h0, err := t.Get(k) + if err != nil { + return false, err + } + + var h int64 + if len(h0) != 0 { + h, _ = h0[0].(int64) + } + nh, err := t.Create(append([]interface{}{h, nil}, in...)...) + if err != nil { + return false, err + } + + for i, c := range gcols { + k[i] = in[c.index] + } + err = t.Set(k, []interface{}{nh}) + if err != nil { + return false, err + } + + return true, nil + } + + ok = true + flds = in[0].([]*fld) + for _, c := range r.colNames { + i := findFldIndex(flds, c) + if i < 0 { + return false, fmt.Errorf("unknown column %s", c) + } + + gcols = append(gcols, &col{name: c, index: i}) + } + return !onlyNames, nil + }); err != nil { + return + } + + if onlyNames { + _, err := f(nil, []interface{}{flds}) + return err + } + + it, err := t.SeekFirst() + if err != nil { + return noEOF(err) + } + + for i, v := range flds { + cols[i].name = v.name + cols[i].index = i + } + + var data []interface{} + var more bool + for more, err = f(nil, []interface{}{t, cols}); more && err == nil; more, err = f(nil, data) { + _, data, err = it.Next() + if err != nil { + return noEOF(err) + } + } + return err +} + +// TCtx represents transaction context. It enables to execute multiple +// statement lists in the same context. The same context guarantees the state +// of the DB cannot change in between the separated executions. +// +// LastInsertID +// +// LastInsertID is updated by INSERT INTO statements. The value considers +// performed ROLLBACK statements, if any, even though roll backed IDs are not +// reused. QL clients should treat the field as read only. +// +// RowsAffected +// +// RowsAffected is updated by INSERT INTO, DELETE FROM and UPDATE statements. +// The value does not (yet) consider any ROLLBACK statements involved. QL +// clients should treat the field as read only. +type TCtx struct { + LastInsertID int64 + RowsAffected int64 +} + +// NewRWCtx returns a new read/write transaction context. NewRWCtx is safe for +// concurrent use by multiple goroutines, every one of them will get a new, +// unique conext. +func NewRWCtx() *TCtx { return &TCtx{} } + +// Recordset is a result of a select statment. It can call a user function for +// every row (record) in the set using the Do method. +// +// Recordsets can be safely reused. Evaluation of the rows is performed lazily. +// Every invocation of Do will see the current, potentially actualized data. +// +// Do +// +// Do will call f for every row (record) in the Recordset. +// +// If f returns more == false or err != nil then f will not be called for any +// remaining rows in the set and the err value is returned from Do. +// +// If names == true then f is firstly called with a virtual row +// consisting of field (column) names of the RecordSet. +// +// Do is executed in a read only context and performs a RLock of the +// database. +// +// Do is safe for concurrent use by multiple goroutines. +// +// Fields +// +// The only reliable way, in the general case, how to get field names of a +// recordset is to execute the Do method with the names parameter set to true. +// Any SELECT can return different fields on different runs, provided the +// columns of some of the underlying tables involved were altered in between +// and the query sports the SELECT * form. Then the fields are not really +// known until the first query result row materializes. The problem is that +// some queries can be costly even before that first row is computed. If only +// the field names is what is required in some situation then executing such +// costly query could be prohibitively expensive. +// +// The Fields method provides an alternative. It computes the recordset fields +// while ignoring table data, WHERE clauses, predicates and without evaluating +// any expressions nor any functions. +// +// The result of Fields can be obviously imprecise if tables are altered before +// running Do later. In exchange, calling Fields is cheap - compared to +// actually computing a first row of a query having, say cross joins on n +// relations (1^n is always 1, n ∈ N). +// +// FirstRow +// +// FirstRow will return the first row of the RecordSet or an error, if any. If +// the Recordset has no rows the result is (nil, nil). +// +// Rows +// +// Rows will return rows in Recordset or an error, if any. The semantics of +// limit and offset are the same as of the LIMIT and OFFSET clauses of the +// SELECT statement. To get all rows pass limit < 0. If there are no rows to +// return the result is (nil, nil). +type Recordset interface { + Do(names bool, f func(data []interface{}) (more bool, err error)) error + Fields() (names []string, err error) + FirstRow() (row []interface{}, err error) + Rows(limit, offset int) (rows [][]interface{}, err error) +} + +type assignment struct { + colName string + expr expression +} + +func (a *assignment) String() string { + return fmt.Sprintf("%s=%s", a.colName, a.expr) +} + +type distinctRset struct { + src rset +} + +func (r *distinctRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + t, err := ctx.db.store.CreateTemp(true) + if err != nil { + return + } + + defer func() { + if derr := t.Drop(); derr != nil && err == nil { + err = derr + } + }() + + var flds []*fld + ok := false + if err = r.src.do(ctx, onlyNames, func(id interface{}, in []interface{}) (more bool, err error) { + if ok { + if err = t.Set(in, nil); err != nil { + return false, err + } + + return true, nil + } + + flds = in[0].([]*fld) + ok = true + return true && !onlyNames, nil + }); err != nil { + return + } + + if onlyNames { + _, err := f(nil, []interface{}{flds}) + return noEOF(err) + } + + it, err := t.SeekFirst() + if err != nil { + return noEOF(err) + } + + var data []interface{} + var more bool + for more, err = f(nil, []interface{}{flds}); more && err == nil; more, err = f(nil, data) { + data, _, err = it.Next() + if err != nil { + return noEOF(err) + } + } + return err +} + +type orderByRset struct { + asc bool + by []expression + src rset +} + +func (r *orderByRset) String() string { + a := make([]string, len(r.by)) + for i, v := range r.by { + a[i] = v.String() + } + s := strings.Join(a, ", ") + if !r.asc { + s += " DESC" + } + return s +} + +func (r *orderByRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + t, err := ctx.db.store.CreateTemp(r.asc) + if err != nil { + return + } + + defer func() { + if derr := t.Drop(); derr != nil && err == nil { + err = derr + } + }() + + m := map[interface{}]interface{}{} + var flds []*fld + ok := false + k := make([]interface{}, len(r.by)+1) + id := int64(-1) + if err = r.src.do(ctx, onlyNames, func(rid interface{}, in []interface{}) (more bool, err error) { + id++ + if ok { + for i, fld := range flds { + if nm := fld.name; nm != "" { + m[nm] = in[i] + } + } + m["$id"] = rid + for i, expr := range r.by { + val, err := expr.eval(ctx, m, ctx.arg) + if err != nil { + return false, err + } + + if val != nil { + val, ordered, err := isOrderedType(val) + if err != nil { + return false, err + } + + if !ordered { + return false, fmt.Errorf("cannot order by %v (type %T)", val, val) + + } + } + + k[i] = val + } + k[len(r.by)] = id + if err = t.Set(k, in); err != nil { + return false, err + } + + return true, nil + } + + ok = true + flds = in[0].([]*fld) + return true && !onlyNames, nil + }); err != nil { + return + } + + if onlyNames { + _, err = f(nil, []interface{}{flds}) + return noEOF(err) + } + + it, err := t.SeekFirst() + if err != nil { + if err != io.EOF { + return err + } + + _, err = f(nil, []interface{}{flds}) + return err + } + + var data []interface{} + var more bool + for more, err = f(nil, []interface{}{flds}); more && err == nil; more, err = f(nil, data) { + _, data, err = it.Next() + if err != nil { + return noEOF(err) + } + } + return +} + +var nowhere = &whereRset{} + +type whereRset struct { + expr expression + src rset +} + +func (r *whereRset) doIndexedBool(t *table, en indexIterator, v bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + m, err := f(nil, []interface{}{t.flds()}) + if !m || err != nil { + return + } + + for { + k, h, err := en.Next() + if err != nil { + return noEOF(err) + } + + switch x := k.(type) { + case nil: + panic("internal error 052") // nil should sort before true + case bool: + if x != v { + return nil + } + } + + if _, err := tableRset("").doOne(t, h, f); err != nil { + return err + } + } +} + +func (r *whereRset) tryBinOp(execCtx *execCtx, t *table, id *ident, v value, op int, f func(id interface{}, data []interface{}) (more bool, err error)) (bool, error) { + c := findCol(t.cols0, id.s) + if c == nil { + return false, fmt.Errorf("undefined column: %s", id.s) + } + + xCol := t.indices[c.index+1] + if xCol == nil { // no index for this column + return false, nil + } + + data := []interface{}{v.val} + cc := *c + cc.index = 0 + if err := typeCheck(data, []*col{&cc}); err != nil { + return true, err + } + + v.val = data[0] + ex := &binaryOperation{op, nil, v} + switch op { + case '<', le: + v.val = false // first value collating after nil + fallthrough + case eq, ge: + m, err := f(nil, []interface{}{t.flds()}) + if !m || err != nil { + return true, err + } + + en, _, err := xCol.x.Seek(v.val) + if err != nil { + return true, noEOF(err) + } + + for { + k, h, err := en.Next() + if k == nil { + return true, nil + } + + if err != nil { + return true, noEOF(err) + } + + ex.l = value{k} + eval, err := ex.eval(execCtx, nil, nil) + if err != nil { + return true, err + } + + if !eval.(bool) { + return true, nil + } + + if _, err := tableRset("").doOne(t, h, f); err != nil { + return true, err + } + } + case '>': + m, err := f(nil, []interface{}{t.flds()}) + if !m || err != nil { + return true, err + } + + en, err := xCol.x.SeekLast() + if err != nil { + return true, noEOF(err) + } + + for { + k, h, err := en.Prev() + if k == nil { + return true, nil + } + + if err != nil { + return true, noEOF(err) + } + + ex.l = value{k} + eval, err := ex.eval(execCtx, nil, nil) + if err != nil { + return true, err + } + + if !eval.(bool) { + return true, nil + } + + if _, err := tableRset("").doOne(t, h, f); err != nil { + return true, err + } + } + default: + panic("internal error 053") + } +} + +func (r *whereRset) tryBinOpID(execCtx *execCtx, t *table, v value, op int, f func(id interface{}, data []interface{}) (more bool, err error)) (bool, error) { + xCol := t.indices[0] + if xCol == nil { // no index for id() + return false, nil + } + + data := []interface{}{v.val} + if err := typeCheck(data, []*col{&col{typ: qInt64}}); err != nil { + return true, err + } + + v.val = data[0] + ex := &binaryOperation{op, nil, v} + switch op { + case '<', le: + v.val = int64(1) + fallthrough + case eq, ge: + m, err := f(nil, []interface{}{t.flds()}) + if !m || err != nil { + return true, err + } + + en, _, err := xCol.x.Seek(v.val) + if err != nil { + return true, noEOF(err) + } + + for { + k, h, err := en.Next() + if k == nil { + return true, nil + } + + if err != nil { + return true, noEOF(err) + } + + ex.l = value{k} + eval, err := ex.eval(execCtx, nil, nil) + if err != nil { + return true, err + } + + if !eval.(bool) { + return true, nil + } + + if _, err := tableRset("").doOne(t, h, f); err != nil { + return true, err + } + } + case '>': + m, err := f(nil, []interface{}{t.flds()}) + if !m || err != nil { + return true, err + } + + en, err := xCol.x.SeekLast() + if err != nil { + return true, noEOF(err) + } + + for { + k, h, err := en.Prev() + if k == nil { + return true, nil + } + + if err != nil { + return true, noEOF(err) + } + + ex.l = value{k} + eval, err := ex.eval(execCtx, nil, nil) + if err != nil { + return true, err + } + + if !eval.(bool) { + return true, nil + } + + if _, err := tableRset("").doOne(t, h, f); err != nil { + return true, err + } + } + default: + panic("internal error 071") + } +} + +func (r *whereRset) tryUseIndex(ctx *execCtx, f func(id interface{}, data []interface{}) (more bool, err error)) (bool, error) { + //TODO(indices) support IS [NOT] NULL + c, ok := r.src.(*crossJoinRset) + if !ok { + return false, nil + } + + tabName, ok := c.isSingleTable() + if !ok || isSystemName[tabName] { + return false, nil + } + + t := ctx.db.root.tables[tabName] + if t == nil { + return true, fmt.Errorf("table %s does not exist", tabName) + } + + if !t.hasIndices() { + return false, nil + } + + //LATER WHERE column1 boolOp column2 ... + //LATER WHERE !column (rewritable as: column == false) + switch ex := r.expr.(type) { + case *unaryOperation: // WHERE !column + if ex.op != '!' { + return false, nil + } + + switch operand := ex.v.(type) { + case *ident: + c := findCol(t.cols0, operand.s) + if c == nil { // no such column + return false, fmt.Errorf("unknown column %s", ex) + } + + if c.typ != qBool { // not a bool column + return false, nil + } + + xCol := t.indices[c.index+1] + if xCol == nil { // column isn't indexed + return false, nil + } + + en, _, err := xCol.x.Seek(false) + if err != nil { + return false, noEOF(err) + } + + return true, r.doIndexedBool(t, en, false, f) + default: + return false, nil + } + case *ident: // WHERE column + c := findCol(t.cols0, ex.s) + if c == nil { // no such column + return false, fmt.Errorf("unknown column %s", ex) + } + + if c.typ != qBool { // not a bool column + return false, nil + } + + xCol := t.indices[c.index+1] + if xCol == nil { // column isn't indexed + return false, nil + } + + en, _, err := xCol.x.Seek(true) + if err != nil { + return false, noEOF(err) + } + + return true, r.doIndexedBool(t, en, true, f) + case *binaryOperation: + //DONE handle id() + var invOp int + switch ex.op { + case '<': + invOp = '>' + case le: + invOp = ge + case eq: + invOp = eq + case '>': + invOp = '<' + case ge: + invOp = le + default: + return false, nil + } + + switch lhs := ex.l.(type) { + case *call: + if !(lhs.f == "id" && len(lhs.arg) == 0) { + return false, nil + } + + switch rhs := ex.r.(type) { + case parameter: + v, err := rhs.eval(ctx, nil, ctx.arg) + if err != nil { + return false, err + } + + return r.tryBinOpID(ctx, t, value{v}, ex.op, f) + case value: + return r.tryBinOpID(ctx, t, rhs, ex.op, f) + default: + return false, nil + } + case *ident: + switch rhs := ex.r.(type) { + case parameter: + v, err := rhs.eval(ctx, nil, ctx.arg) + if err != nil { + return false, err + } + + return r.tryBinOp(ctx, t, lhs, value{v}, ex.op, f) + case value: + return r.tryBinOp(ctx, t, lhs, rhs, ex.op, f) + default: + return false, nil + } + case parameter: + switch rhs := ex.r.(type) { + case *call: + if !(rhs.f == "id" && len(rhs.arg) == 0) { + return false, nil + } + + v, err := lhs.eval(ctx, nil, ctx.arg) + if err != nil { + return false, err + } + + return r.tryBinOpID(ctx, t, value{v}, invOp, f) + case *ident: + v, err := lhs.eval(ctx, nil, ctx.arg) + if err != nil { + return false, err + } + + return r.tryBinOp(ctx, t, rhs, value{v}, invOp, f) + default: + return false, nil + } + case value: + switch rhs := ex.r.(type) { + case *call: + if !(rhs.f == "id" && len(rhs.arg) == 0) { + return false, nil + } + + return r.tryBinOpID(ctx, t, lhs, invOp, f) + case *ident: + return r.tryBinOp(ctx, t, rhs, lhs, invOp, f) + default: + return false, nil + } + default: + return false, nil + } + default: + return false, nil + } +} + +func (r *whereRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + //dbg("====") + if !onlyNames { + if ok, err := r.tryUseIndex(ctx, f); ok || err != nil { + //dbg("ok %t, err %v", ok, err) + return err + } + } + + //dbg("not using indices") + m := map[interface{}]interface{}{} + var flds []*fld + ok := false + return r.src.do(ctx, onlyNames, func(rid interface{}, in []interface{}) (more bool, err error) { + if ok { + for i, fld := range flds { + if nm := fld.name; nm != "" { + m[nm] = in[i] + } + } + m["$id"] = rid + val, err := r.expr.eval(ctx, m, ctx.arg) + if err != nil { + return false, err + } + + if val == nil { + return true, nil + } + + x, ok := val.(bool) + if !ok { + return false, fmt.Errorf("invalid WHERE expression %s (value of type %T)", val, val) + } + + if !x { + return true, nil + } + + return f(rid, in) + } + + flds = in[0].([]*fld) + ok = true + m, err := f(nil, in) + return m && !onlyNames, err + }) +} + +type offsetRset struct { + expr expression + src rset +} + +func (r *offsetRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + m := map[interface{}]interface{}{} + var flds []*fld + var ok, eval bool + var off uint64 + return r.src.do(ctx, onlyNames, func(rid interface{}, in []interface{}) (more bool, err error) { + if ok { + if !eval { + for i, fld := range flds { + if nm := fld.name; nm != "" { + m[nm] = in[i] + } + } + m["$id"] = rid + val, err := r.expr.eval(ctx, m, ctx.arg) + if err != nil { + return false, err + } + + if val == nil { + return true, nil + } + + if off, err = limOffExpr(val); err != nil { + return false, err + } + + eval = true + } + if off > 0 { + off-- + return true, nil + } + + return f(rid, in) + } + + flds = in[0].([]*fld) + ok = true + m, err := f(nil, in) + return m && !onlyNames, err + }) +} + +type limitRset struct { + expr expression + src rset +} + +func (r *limitRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + m := map[interface{}]interface{}{} + var flds []*fld + var ok, eval bool + var lim uint64 + return r.src.do(ctx, onlyNames, func(rid interface{}, in []interface{}) (more bool, err error) { + if ok { + if !eval { + for i, fld := range flds { + if nm := fld.name; nm != "" { + m[nm] = in[i] + } + } + m["$id"] = rid + val, err := r.expr.eval(ctx, m, ctx.arg) + if err != nil { + return false, err + } + + if val == nil { + return true, nil + } + + if lim, err = limOffExpr(val); err != nil { + return false, err + } + + eval = true + } + switch lim { + case 0: + return false, nil + default: + lim-- + return f(rid, in) + } + } + + flds = in[0].([]*fld) + ok = true + m, err := f(nil, in) + return m && !onlyNames, err + }) +} + +type selectRset struct { + flds []*fld + src rset +} + +func (r *selectRset) doGroup(grp *groupByRset, ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + if onlyNames { + if len(r.flds) != 0 { + _, err := f(nil, []interface{}{r.flds}) + return err + } + + return grp.do(ctx, true, f) + } + + var t temp + var cols []*col + out := make([]interface{}, len(r.flds)) + ok := false + rows := 0 + if err = r.src.do(ctx, onlyNames, func(rid interface{}, in []interface{}) (more bool, err error) { + if ok { + h := in[0].(int64) + m := map[interface{}]interface{}{} + for h != 0 { + in, err = t.Read(nil, h, cols...) + if err != nil { + return false, err + } + + rec := in[2:] + for i, c := range cols { + if nm := c.name; nm != "" { + m[nm] = rec[i] + } + } + m["$id"] = rid + for _, fld := range r.flds { + if _, err = fld.expr.eval(ctx, m, ctx.arg); err != nil { + return false, err + } + } + + h = in[0].(int64) + } + m["$agg"] = true + for i, fld := range r.flds { + if out[i], err = fld.expr.eval(ctx, m, ctx.arg); err != nil { + return false, err + } + } + rows++ + return f(nil, out) + } + + ok = true + rows++ + t = in[0].(temp) + cols = in[1].([]*col) + if len(r.flds) == 0 { + r.flds = make([]*fld, len(cols)) + for i, v := range cols { + r.flds[i] = &fld{expr: &ident{v.name}, name: v.name} + } + out = make([]interface{}, len(r.flds)) + } + m, err := f(nil, []interface{}{r.flds}) + return m && !onlyNames, err + }); err != nil || onlyNames { + return + } + + switch rows { + case 0: + more, err := f(nil, []interface{}{r.flds}) + if !more || err != nil { + return err + } + + fallthrough + case 1: + m := map[interface{}]interface{}{"$agg0": true} // aggregate empty record set + for i, fld := range r.flds { + if out[i], err = fld.expr.eval(ctx, m, ctx.arg); err != nil { + return + } + } + _, err = f(nil, out) + } + return +} + +func (r *selectRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + if grp, ok := r.src.(*groupByRset); ok { + return r.doGroup(grp, ctx, onlyNames, f) + } + + if len(r.flds) == 0 { + return r.src.do(ctx, onlyNames, f) + } + + if onlyNames { + _, err := f(nil, []interface{}{r.flds}) + return err + } + + var flds []*fld + m := map[interface{}]interface{}{} + ok := false + return r.src.do(ctx, onlyNames, func(rid interface{}, in []interface{}) (more bool, err error) { + if ok { + for i, fld := range flds { + if nm := fld.name; nm != "" { + m[nm] = in[i] + } + } + m["$id"] = rid + out := make([]interface{}, len(r.flds)) + for i, fld := range r.flds { + if out[i], err = fld.expr.eval(ctx, m, ctx.arg); err != nil { + return false, err + } + } + m, err := f(rid, out) + return m, err + + } + + ok = true + flds = in[0].([]*fld) + m, err := f(nil, []interface{}{r.flds}) + return m && !onlyNames, err + }) +} + +type tableRset string + +func (r tableRset) doIndex(x *indexedCol, ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + flds := []*fld{&fld{name: x.name}} + m, err := f(nil, []interface{}{flds}) + if onlyNames { + return err + } + + if !m || err != nil { + return + } + + en, _, err := x.x.Seek(nil) + if err != nil { + return err + } + + var id int64 + rec := []interface{}{nil} + for { + k, _, err := en.Next() + if err != nil { + return noEOF(err) + } + + id++ + rec[0] = k + m, err := f(id, rec) + if !m || err != nil { + return err + } + } +} + +func (tableRset) doOne(t *table, h int64, f func(id interface{}, data []interface{}) (more bool, err error)) ( /* next handle */ int64, error) { + cols := t.cols + ncols := len(cols) + rec, err := t.store.Read(nil, h, cols...) + if err != nil { + return -1, err + } + + h = rec[0].(int64) + if n := ncols + 2 - len(rec); n > 0 { + rec = append(rec, make([]interface{}, n)...) + } + + for i, c := range cols { + if x := c.index; 2+x < len(rec) { + rec[2+i] = rec[2+x] + continue + } + + rec[2+i] = nil //DONE +test (#571) + } + m, err := f(rec[1], rec[2:2+ncols]) // 0:next, 1:id + if !m || err != nil { + return -1, err + } + + return h, nil +} + +func (r tableRset) doSysTable(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + flds := []*fld{&fld{name: "Name"}, &fld{name: "Schema"}} + m, err := f(nil, []interface{}{flds}) + if onlyNames { + return err + } + + if !m || err != nil { + return + } + + rec := make([]interface{}, 2) + di, err := ctx.db.info() + if err != nil { + return err + } + + var id int64 + for _, ti := range di.Tables { + rec[0] = ti.Name + a := []string{} + for _, ci := range ti.Columns { + a = append(a, fmt.Sprintf("%s %s", ci.Name, ci.Type)) + } + rec[1] = fmt.Sprintf("CREATE TABLE %s (%s);", ti.Name, strings.Join(a, ", ")) + id++ + m, err := f(id, rec) + if !m || err != nil { + return err + } + } + return +} + +func (r tableRset) doSysColumn(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + flds := []*fld{&fld{name: "TableName"}, &fld{name: "Ordinal"}, &fld{name: "Name"}, &fld{name: "Type"}} + m, err := f(nil, []interface{}{flds}) + if onlyNames { + return err + } + + if !m || err != nil { + return + } + + rec := make([]interface{}, 4) + di, err := ctx.db.info() + if err != nil { + return err + } + + var id int64 + for _, ti := range di.Tables { + rec[0] = ti.Name + var ix int64 + for _, ci := range ti.Columns { + ix++ + rec[1] = ix + rec[2] = ci.Name + rec[3] = ci.Type.String() + id++ + m, err := f(id, rec) + if !m || err != nil { + return err + } + } + } + return +} + +func (r tableRset) doSysIndex(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + flds := []*fld{&fld{name: "TableName"}, &fld{name: "ColumnName"}, &fld{name: "Name"}, &fld{name: "IsUnique"}} + m, err := f(nil, []interface{}{flds}) + if onlyNames { + return err + } + + if !m || err != nil { + return + } + + rec := make([]interface{}, 4) + di, err := ctx.db.info() + if err != nil { + return err + } + + var id int64 + for _, xi := range di.Indices { + rec[0] = xi.Table + rec[1] = xi.Column + rec[2] = xi.Name + rec[3] = xi.Unique + id++ + m, err := f(id, rec) + if !m || err != nil { + return err + } + } + return +} + +func (r tableRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + switch r { + case "__Table": + return r.doSysTable(ctx, onlyNames, f) + case "__Column": + return r.doSysColumn(ctx, onlyNames, f) + case "__Index": + return r.doSysIndex(ctx, onlyNames, f) + } + + t, ok := ctx.db.root.tables[string(r)] + var x *indexedCol + if !ok && isTesting { + if _, x = ctx.db.root.findIndexByName(string(r)); x != nil { + return r.doIndex(x, ctx, onlyNames, f) + } + } + + if !ok { + return fmt.Errorf("table %s does not exist", r) + } + + m, err := f(nil, []interface{}{t.flds()}) + if onlyNames { + return err + } + + if !m || err != nil { + return + } + + for h := t.head; h > 0 && err == nil; h, err = r.doOne(t, h, f) { + } + return +} + +type crossJoinRset struct { + sources []interface{} +} + +func (r *crossJoinRset) tables() []struct { + i int + name, rename string +} { + var ret []struct { + i int + name, rename string + } + //dbg("---- %p", r) + for i, pair0 := range r.sources { + //dbg("%d/%d, %#v", i, len(r.sources), pair0) + pair := pair0.([]interface{}) + altName := pair[1].(string) + switch x := pair[0].(type) { + case string: // table name + if altName == "" { + altName = x + } + ret = append(ret, struct { + i int + name, rename string + }{i, x, altName}) + } + } + return ret +} + +func (r *crossJoinRset) String() string { + a := make([]string, len(r.sources)) + for i, pair0 := range r.sources { + pair := pair0.([]interface{}) + altName := pair[1].(string) + switch x := pair[0].(type) { + case string: // table name + switch { + case altName == "": + a[i] = x + default: + a[i] = fmt.Sprintf("%s AS %s", x, altName) + } + case *selectStmt: + switch { + case altName == "": + a[i] = fmt.Sprintf("(%s)", x) + default: + a[i] = fmt.Sprintf("(%s) AS %s", x, altName) + } + default: + log.Panic("internal error 054") + } + } + return strings.Join(a, ", ") +} + +func (r *crossJoinRset) isSingleTable() (string, bool) { + sources := r.sources + if len(sources) != 1 { + return "", false + } + + pair := sources[0].([]interface{}) + s, ok := pair[0].(string) + return s, ok +} + +func (r *crossJoinRset) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + rsets := make([]rset, len(r.sources)) + altNames := make([]string, len(r.sources)) + //dbg(".... %p", r) + for i, pair0 := range r.sources { + pair := pair0.([]interface{}) + //dbg("%d: %#v", len(pair), pair) + altName := pair[1].(string) + switch x := pair[0].(type) { + case string: // table name + rsets[i] = tableRset(x) + if altName == "" { + altName = x + } + case *selectStmt: + rsets[i] = x + default: + log.Panic("internal error 055") + } + altNames[i] = altName + } + + if len(rsets) == 1 { + return rsets[0].do(ctx, onlyNames, f) + } + + var flds []*fld + fldsSent := false + iq := 0 + stop := false + ids := map[string]interface{}{} + var g func([]interface{}, []rset, int) error + g = func(prefix []interface{}, rsets []rset, x int) (err error) { + rset := rsets[0] + rsets = rsets[1:] + ok := false + return rset.do(ctx, onlyNames, func(id interface{}, in []interface{}) (more bool, err error) { + if onlyNames && fldsSent { + stop = true + return false, nil + } + + if ok { + ids[altNames[x]] = id + if len(rsets) != 0 { + return true, g(append(prefix, in...), rsets, x+1) + } + + m, err := f(ids, append(prefix, in...)) + if !m { + stop = true + } + return m && !stop, err + } + + ok = true + if !fldsSent { + f0 := append([]*fld(nil), in[0].([]*fld)...) + q := altNames[iq] + for i, elem := range f0 { + nf := &fld{} + *nf = *elem + switch { + case q == "": + nf.name = "" + case nf.name != "": + nf.name = fmt.Sprintf("%s.%s", altNames[iq], nf.name) + } + f0[i] = nf + } + iq++ + flds = append(flds, f0...) + } + if len(rsets) == 0 && !fldsSent { + fldsSent = true + more, err = f(nil, []interface{}{flds}) + if !more { + stop = true + } + return more && !stop, err + } + + return !stop, nil + }) + } + return g(nil, rsets, 0) +} + +type fld struct { + expr expression + name string +} + +func findFldIndex(fields []*fld, name string) int { + for i, f := range fields { + if f.name == name { + return i + } + } + + return -1 +} + +func findFld(fields []*fld, name string) (f *fld) { + for _, f = range fields { + if f.name == name { + return + } + } + + return nil +} + +type col struct { + index int + name string + typ int +} + +func findCol(cols []*col, name string) (c *col) { + for _, c = range cols { + if c.name == name { + return + } + } + + return nil +} + +func (f *col) typeCheck(x interface{}) (ok bool) { //NTYPE + switch x.(type) { + case nil: + return true + case bool: + return f.typ == qBool + case complex64: + return f.typ == qComplex64 + case complex128: + return f.typ == qComplex128 + case float32: + return f.typ == qFloat32 + case float64: + return f.typ == qFloat64 + case int8: + return f.typ == qInt8 + case int16: + return f.typ == qInt16 + case int32: + return f.typ == qInt32 + case int64: + return f.typ == qInt64 + case string: + return f.typ == qString + case uint8: + return f.typ == qUint8 + case uint16: + return f.typ == qUint16 + case uint32: + return f.typ == qUint32 + case uint64: + return f.typ == qUint64 + case []byte: + return f.typ == qBlob + case *big.Int: + return f.typ == qBigInt + case *big.Rat: + return f.typ == qBigRat + case time.Time: + return f.typ == qTime + case time.Duration: + return f.typ == qDuration + case chunk: + return true // was checked earlier + } + return +} + +func cols2meta(f []*col) (s string) { + a := []string{} + for _, f := range f { + a = append(a, string(f.typ)+f.name) + } + return strings.Join(a, "|") +} + +// DB represent the database capable of executing QL statements. +type DB struct { + cc *TCtx // Current transaction context + isMem bool + mu sync.Mutex + root *root + rw bool // DB FSM + rwmu sync.RWMutex + store storage + tnl int // Transaction nesting level +} + +func newDB(store storage) (db *DB, err error) { + db0 := &DB{ + store: store, + } + if db0.root, err = newRoot(store); err != nil { + return + } + + return db0, nil +} + +// Name returns the name of the DB. +func (db *DB) Name() string { return db.store.Name() } + +// Run compiles and executes a statement list. It returns, if applicable, a +// RecordSet slice and/or an index and error. +// +// For more details please see DB.Execute +// +// Run is safe for concurrent use by multiple goroutines. +func (db *DB) Run(ctx *TCtx, ql string, arg ...interface{}) (rs []Recordset, index int, err error) { + l, err := Compile(ql) + if err != nil { + return nil, -1, err + } + + return db.Execute(ctx, l, arg...) +} + +// Compile parses the ql statements from src and returns a compiled list for +// DB.Execute or an error if any. +// +// Compile is safe for concurrent use by multiple goroutines. +func Compile(src string) (List, error) { + l := newLexer(src) + if yyParse(l) != 0 { + return List{}, l.errs[0] + } + + return List{l.list, l.params}, nil +} + +// MustCompile is like Compile but panics if the ql statements in src cannot be +// compiled. It simplifies safe initialization of global variables holding +// compiled statement lists for DB.Execute. +// +// MustCompile is safe for concurrent use by multiple goroutines. +func MustCompile(src string) List { + list, err := Compile(src) + if err != nil { + panic("ql: Compile(" + strconv.Quote(src) + "): " + err.Error()) // panic ok here + } + + return list +} + +// Execute executes statements in a list while substituting QL paramaters from +// arg. +// +// The resulting []Recordset corresponds to the SELECT FROM statements in the +// list. +// +// If err != nil then index is the zero based index of the failed QL statement. +// Empty statements do not count. +// +// The FSM STT describing the relations between DB states, statements and the +// ctx parameter. +// +// +-----------+---------------------+------------------+------------------+------------------+ +// |\ Event | | | | | +// | \-------\ | BEGIN | | | Other | +// | State \| TRANSACTION | COMMIT | ROLLBACK | statement | +// +-----------+---------------------+------------------+------------------+------------------+ +// | RD | if PC == nil | return error | return error | DB.RLock | +// | | return error | | | Execute(1) | +// | CC == nil | | | | DB.RUnlock | +// | TNL == 0 | DB.Lock | | | | +// | | CC = PC | | | | +// | | TNL++ | | | | +// | | DB.BeginTransaction | | | | +// | | State = WR | | | | +// +-----------+---------------------+------------------+------------------+------------------+ +// | WR | if PC == nil | if PC != CC | if PC != CC | if PC == nil | +// | | return error | return error | return error | DB.Rlock | +// | CC != nil | | | | Execute(1) | +// | TNL != 0 | if PC != CC | DB.Commit | DB.Rollback | RUnlock | +// | | DB.Lock | TNL-- | TNL-- | else if PC != CC | +// | | CC = PC | if TNL == 0 | if TNL == 0 | return error | +// | | | CC = nil | CC = nil | else | +// | | TNL++ | State = RD | State = RD | Execute(2) | +// | | DB.BeginTransaction | DB.Unlock | DB.Unlock | | +// +-----------+---------------------+------------------+------------------+------------------+ +// CC: Curent transaction context +// PC: Passed transaction context +// TNL: Transaction nesting level +// +// Lock, Unlock, RLock, RUnlock semantics above are the same as in +// sync.RWMutex. +// +// (1): Statement list is executed outside of a transaction. Attempts to update +// the DB will fail, the execution context is read-only. Other statements with +// read only context will execute concurrently. If any statement fails, the +// execution of the statement list is aborted. +// +// Note that the RLock/RUnlock surrounds every single "other" statement when it +// is executed outside of a transaction. If read consistency is required by a +// list of more than one statement then an explicit BEGIN TRANSACTION / COMMIT +// or ROLLBACK wrapper must be provided. Otherwise the state of the DB may +// change in between executing any two out-of-transaction statements. +// +// (2): Statement list is executed inside an isolated transaction. Execution of +// statements can update the DB, the execution context is read-write. If any +// statement fails, the execution of the statement list is aborted and the DB +// is automatically rolled back to the TNL which was active before the start of +// execution of the statement list. +// +// Execute is safe for concurrent use by multiple goroutines, but one must +// consider the blocking issues as discussed above. +// +// ACID +// +// Atomicity: Transactions are atomic. Transactions can be nested. Commit or +// rollbacks work on the current transaction level. Transactions are made +// persistent only on the top level commit. Reads made from within an open +// transaction are dirty reads. +// +// Consistency: Transactions bring the DB from one structurally consistent +// state to other structurally consistent state. +// +// Isolation: Transactions are isolated. Isolation is implemented by +// serialization. +// +// Durability: Transactions are durable. A two phase commit protocol and a +// write ahead log is used. Database is recovered after a crash from the write +// ahead log automatically on open. +func (db *DB) Execute(ctx *TCtx, l List, arg ...interface{}) (rs []Recordset, index int, err error) { + // Sanitize args + for i, v := range arg { + switch x := v.(type) { + case nil, bool, complex64, complex128, float32, float64, string, + int8, int16, int32, int64, int, + uint8, uint16, uint32, uint64, uint, + *big.Int, *big.Rat, []byte, time.Duration, time.Time: + case big.Int: + arg[i] = &x + case big.Rat: + arg[i] = &x + default: + return nil, 0, fmt.Errorf("cannot use arg[%d] (type %T):unsupported type", i, v) + } + } + + tnl0 := -1 + if ctx != nil { + ctx.LastInsertID, ctx.RowsAffected = 0, 0 + } + + var s stmt + for index, s = range l.l { + r, err := db.run1(ctx, &tnl0, s, arg...) + if err != nil { + for tnl0 >= 0 && db.tnl > tnl0 { + if _, e2 := db.run1(ctx, &tnl0, rollbackStmt{}); e2 != nil { + err = e2 + } + } + return rs, index, err + } + + if r != nil { + rs = append(rs, r) + } + } + return +} + +func (db *DB) run1(pc *TCtx, tnl0 *int, s stmt, arg ...interface{}) (rs Recordset, err error) { + //dbg("%v", s) + db.mu.Lock() + switch db.rw { + case false: + switch s.(type) { + case beginTransactionStmt: + defer db.mu.Unlock() + if pc == nil { + return nil, errors.New("BEGIN TRANSACTION: cannot start a transaction in nil TransactionCtx") + } + + if err = db.store.BeginTransaction(); err != nil { + return + } + + db.beginTransaction() + db.rwmu.Lock() + db.cc = pc + *tnl0 = db.tnl // 0 + db.tnl++ + db.rw = true + return + case commitStmt: + defer db.mu.Unlock() + return nil, errCommitNotInTransaction + case rollbackStmt: + defer db.mu.Unlock() + return nil, errRollbackNotInTransaction + default: + if s.isUpdating() { + db.mu.Unlock() + return nil, fmt.Errorf("attempt to update the DB outside of a transaction") + } + + db.rwmu.RLock() // can safely grab before Unlock + db.mu.Unlock() + defer db.rwmu.RUnlock() + return s.exec(&execCtx{db, arg}) // R/O tctx + } + default: // case true: + switch s.(type) { + case beginTransactionStmt: + defer db.mu.Unlock() + + if pc == nil { + return nil, errBeginTransNoCtx + } + + if pc != db.cc { + for db.rw == true { + db.mu.Unlock() // Transaction isolation + db.mu.Lock() + } + + db.rw = true + db.rwmu.Lock() + *tnl0 = db.tnl // 0 + } + + if err = db.store.BeginTransaction(); err != nil { + return + } + + db.beginTransaction() + db.cc = pc + db.tnl++ + return + case commitStmt: + defer db.mu.Unlock() + if pc != db.cc { + return nil, fmt.Errorf("invalid passed transaction context") + } + + db.commit() + err = db.store.Commit() + db.tnl-- + if db.tnl != 0 { + return + } + + db.cc = nil + db.rw = false + db.rwmu.Unlock() + return + case rollbackStmt: + defer db.mu.Unlock() + defer func() { pc.LastInsertID = db.root.lastInsertID }() + if pc != db.cc { + return nil, fmt.Errorf("invalid passed transaction context") + } + + db.rollback() + err = db.store.Rollback() + db.tnl-- + if db.tnl != 0 { + return + } + + db.cc = nil + db.rw = false + db.rwmu.Unlock() + return + default: + if pc == nil { + if s.isUpdating() { + db.mu.Unlock() + return nil, fmt.Errorf("attempt to update the DB outside of a transaction") + } + + db.mu.Unlock() // must Unlock before RLock + db.rwmu.RLock() + defer db.rwmu.RUnlock() + return s.exec(&execCtx{db, arg}) + } + + defer db.mu.Unlock() + defer func() { pc.LastInsertID = db.root.lastInsertID }() + if pc != db.cc { + return nil, fmt.Errorf("invalid passed transaction context") + } + + if !s.isUpdating() { + return s.exec(&execCtx{db, arg}) + } + + if rs, err = s.exec(&execCtx{db, arg}); err != nil { + return + } + + return rs, nil + } + } +} + +// Flush ends the transaction collecting window, if applicable. IOW, if the DB +// is dirty, it schedules a 2PC (WAL + DB file) commit on the next outer most +// DB.Commit or performs it synchronously if there's currently no open +// transaction. +// +// The collecting window is an implementation detail and future versions of +// Flush may become a no operation while keeping the operation semantics. +func (db *DB) Flush() (err error) { + return nil +} + +// Close will close the DB. Successful Close is idempotent. +func (db *DB) Close() error { + db.mu.Lock() + defer db.mu.Unlock() + if db.store == nil { + return nil + } + + if db.tnl != 0 { + return fmt.Errorf("cannot close DB while open transaction exist") + } + + err := db.store.Close() + db.root, db.store = nil, nil + return err +} + +func (db *DB) do(r recordset, names int, f func(data []interface{}) (more bool, err error)) (err error) { + db.mu.Lock() + switch db.rw { + case false: + db.rwmu.RLock() // can safely grab before Unlock + db.mu.Unlock() + defer db.rwmu.RUnlock() + default: // case true: + if r.tx == nil { + db.mu.Unlock() // must Unlock before RLock + db.rwmu.RLock() + defer db.rwmu.RUnlock() + break + } + + defer db.mu.Unlock() + if r.tx != db.cc { + return fmt.Errorf("invalid passed transaction context") + } + } + + ok := false + return r.do(r.ctx, names == onlyNames, func(id interface{}, data []interface{}) (more bool, err error) { + if ok { + if err = expand(data); err != nil { + return + } + + return f(data) + } + + ok = true + done := false + switch names { + case noNames: + return true, nil + case onlyNames: + done = true + fallthrough + default: // returnNames + flds := data[0].([]*fld) + a := make([]interface{}, len(flds)) + for i, v := range flds { + a[i] = v.name + } + more, err := f(a) + return more && !done, err + + } + }) +} + +func (db *DB) beginTransaction() { //TODO Rewrite, must use much smaller undo info! + oldRoot := db.root + newRoot := &root{} + *newRoot = *oldRoot + newRoot.parent = oldRoot + a := make([]*table, 0, len(oldRoot.tables)) + newRoot.tables = make(map[string]*table, len(oldRoot.tables)) + for k, v := range oldRoot.tables { + c := v.clone() + a = append(a, c) + newRoot.tables[k] = c + } + for i := 0; i < len(a)-1; i++ { + l, p := a[i], a[i+1] + l.tnext = p + p.tprev = l + } + if len(a) != 0 { + newRoot.thead = a[0] + } + db.root = newRoot +} + +func (db *DB) rollback() { + db.root = db.root.parent +} + +func (db *DB) commit() { + db.root.parent = db.root.parent.parent +} + +// Type represents a QL type (bigint, int, string, ...) +type Type int + +// Values of ColumnInfo.Type. +const ( + BigInt Type = qBigInt + BigRat = qBigRat + Blob = qBlob + Bool = qBool + Complex128 = qComplex128 + Complex64 = qComplex64 + Duration = qDuration + Float32 = qFloat32 + Float64 = qFloat64 + Int16 = qInt16 + Int32 = qInt32 + Int64 = qInt64 + Int8 = qInt8 + String = qString + Time = qTime + Uint16 = qUint16 + Uint32 = qUint32 + Uint64 = qUint64 + Uint8 = qUint8 +) + +// String implements fmt.Stringer. +func (t Type) String() string { + return typeStr(int(t)) +} + +// ColumnInfo provides meta data describing a table column. +type ColumnInfo struct { + Name string // Column name. + Type Type // Column type (BigInt, BigRat, ...). +} + +// TableInfo provides meta data describing a DB table. +type TableInfo struct { + // Table name. + Name string + + // Table schema. Columns are listed in the order in which they appear + // in the schema. + Columns []ColumnInfo +} + +// IndexInfo provides meta data describing a DB index. It corresponds to the +// statement +// +// CREATE INDEX Name ON Table (Column); +type IndexInfo struct { + Name string // Index name + Table string // Table name. + Column string // Column name. + Unique bool // Wheter the index is unique. +} + +// DbInfo provides meta data describing a DB. +type DbInfo struct { + Name string // DB name. + Tables []TableInfo // Tables in the DB. + Indices []IndexInfo // Indices in the DB. +} + +func (db *DB) info() (r *DbInfo, err error) { + r = &DbInfo{Name: db.Name()} + for nm, t := range db.root.tables { + ti := TableInfo{Name: nm} + for _, c := range t.cols { + ti.Columns = append(ti.Columns, ColumnInfo{Name: c.name, Type: Type(c.typ)}) + } + r.Tables = append(r.Tables, ti) + for i, x := range t.indices { + if x == nil { + continue + } + + var cn string + switch { + case i == 0: + cn = "id()" + default: + cn = t.cols0[i-1].name + } + r.Indices = append(r.Indices, IndexInfo{x.name, nm, cn, x.unique}) + } + } + return +} + +// Info provides meta data describing a DB or an error if any. It locks the DB +// to obtain the result. +func (db *DB) Info() (r *DbInfo, err error) { + db.mu.Lock() + defer db.mu.Unlock() + return db.info() +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/ql.y b/Godeps/_workspace/src/github.com/cznic/ql/ql.y new file mode 100644 index 00000000000..29c50719dab --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/ql.y @@ -0,0 +1,1573 @@ +%{ + +//TODO Put your favorite license here + +// yacc source generated by ebnf2y[1] +// at 2015-02-16 13:21:59.814850051 +0100 CET +// +// $ ebnf2y -o ql.y -oe ql.ebnf -start StatementList -pkg ql -p _ +// +// CAUTION: If this file is a Go source file (*.go), it was generated +// automatically by '$ go tool yacc' from a *.y file - DO NOT EDIT in that case! +// +// [1]: http://github.com/cznic/ebnf2y + +package ql //TODO real package name + +//TODO required only be the demo _dump function +import ( + "bytes" + "fmt" + "strings" + + "github.com/cznic/strutil" +) + +%} + +%union { + item interface{} //TODO insert real field(s) +} + +%token _ANDAND +%token _ANDNOT +%token _EQ +%token _FLOAT_LIT +%token _GE +%token _IDENTIFIER +%token _IMAGINARY_LIT +%token _INT_LIT +%token _LE +%token _LSH +%token _NEQ +%token _OROR +%token _QL_PARAMETER +%token _RSH +%token _RUNE_LIT +%token _STRING_LIT + +%type /*TODO real type(s), if/where applicable */ + _ANDAND + _ANDNOT + _EQ + _FLOAT_LIT + _GE + _IDENTIFIER + _IMAGINARY_LIT + _INT_LIT + _LE + _LSH + _NEQ + _OROR + _QL_PARAMETER + _RSH + _RUNE_LIT + _STRING_LIT + +%token _ADD +%token _ALTER +%token _AND +%token _AS +%token _ASC +%token _BEGIN +%token _BETWEEN +%token _BIGINT +%token _BIGRAT +%token _BLOB +%token _BOOL +%token _BY +%token _BYTE +%token _COLUMN +%token _COMMIT +%token _COMPLEX128 +%token _COMPLEX64 +%token _CREATE +%token _DELETE +%token _DESC +%token _DISTINCT +%token _DROP +%token _DURATION +%token _EXISTS +%token _FALSE +%token _FLOAT +%token _FLOAT32 +%token _FLOAT64 +%token _FROM +%token _GROUPBY +%token _ID +%token _IF +%token _IN +%token _INDEX +%token _INSERT +%token _INT +%token _INT16 +%token _INT32 +%token _INT64 +%token _INT8 +%token _INTO +%token _IS +%token _LIKE +%token _LIMIT +%token _NOT +%token _NULL +%token _OFFSET +%token _ON +%token _OR +%token _ORDER +%token _ROLLBACK +%token _RUNE +%token _SELECT +%token _SET +%token _STRING +%token _TABLE +%token _TIME +%token _TRANSACTION +%token _TRUE +%token _TRUNCATE +%token _UINT +%token _UINT16 +%token _UINT32 +%token _UINT64 +%token _UINT8 +%token _UNIQUE +%token _UPDATE +%token _VALUES +%token _WHERE + +%type /*TODO real type(s), if/where applicable */ + AlterTableStmt + AlterTableStmt1 + Assignment + AssignmentList + AssignmentList1 + AssignmentList2 + BeginTransactionStmt + Call + Call1 + ColumnDef + ColumnName + ColumnNameList + ColumnNameList1 + ColumnNameList2 + CommitStmt + Conversion + CreateIndexStmt + CreateIndexStmt1 + CreateIndexStmt2 + CreateIndexStmt3 + CreateTableStmt + CreateTableStmt1 + CreateTableStmt2 + CreateTableStmt3 + DeleteFromStmt + DeleteFromStmt1 + DropIndexStmt + DropIndexStmt1 + DropTableStmt + DropTableStmt1 + EmptyStmt + Expression + Expression1 + Expression11 + ExpressionList + ExpressionList1 + ExpressionList2 + Factor + Factor1 + Factor11 + Factor2 + Field + Field1 + FieldList + FieldList1 + FieldList2 + GroupByClause + Index + IndexName + InsertIntoStmt + InsertIntoStmt1 + InsertIntoStmt2 + Limit + Literal + Offset + Operand + OrderBy + OrderBy1 + OrderBy11 + Predicate + Predicate1 + Predicate11 + Predicate12 + Predicate13 + PrimaryExpression + PrimaryFactor + PrimaryFactor1 + PrimaryFactor11 + PrimaryTerm + PrimaryTerm1 + PrimaryTerm11 + QualifiedIdent + QualifiedIdent1 + RecordSet + RecordSet1 + RecordSet11 + RecordSet2 + RecordSetList + RecordSetList1 + RecordSetList2 + RollbackStmt + SelectStmt + SelectStmt1 + SelectStmt2 + SelectStmt3 + SelectStmt4 + SelectStmt5 + SelectStmt6 + SelectStmt7 + Slice + Slice1 + Slice2 + Start + Statement + StatementList + StatementList1 + TableName + Term + Term1 + Term11 + TruncateTableStmt + Type + UnaryExpr + UnaryExpr1 + UnaryExpr11 + UpdateStmt + UpdateStmt1 + UpdateStmt2 + Values + Values1 + Values2 + WhereClause + +/*TODO %left, %right, ... declarations */ + +%start Start + +%% + +AlterTableStmt: + _ALTER _TABLE TableName AlterTableStmt1 + { + $$ = []AlterTableStmt{"ALTER", "TABLE", $3, $4} //TODO 1 + } + +AlterTableStmt1: + _ADD ColumnDef + { + $$ = []AlterTableStmt1{"ADD", $2} //TODO 2 + } +| _DROP _COLUMN ColumnName + { + $$ = []AlterTableStmt1{"DROP", "COLUMN", $3} //TODO 3 + } + +Assignment: + ColumnName '=' Expression + { + $$ = []Assignment{$1, "=", $3} //TODO 4 + } + +AssignmentList: + Assignment AssignmentList1 AssignmentList2 + { + $$ = []AssignmentList{$1, $2, $3} //TODO 5 + } + +AssignmentList1: + /* EMPTY */ + { + $$ = []AssignmentList1(nil) //TODO 6 + } +| AssignmentList1 ',' Assignment + { + $$ = append($1.([]AssignmentList1), ",", $3) //TODO 7 + } + +AssignmentList2: + /* EMPTY */ + { + $$ = nil //TODO 8 + } +| ',' + { + $$ = "," //TODO 9 + } + +BeginTransactionStmt: + _BEGIN _TRANSACTION + { + $$ = []BeginTransactionStmt{"BEGIN", "TRANSACTION"} //TODO 10 + } + +Call: + '(' Call1 ')' + { + $$ = []Call{"(", $2, ")"} //TODO 11 + } + +Call1: + /* EMPTY */ + { + $$ = nil //TODO 12 + } +| ExpressionList + { + $$ = $1 //TODO 13 + } + +ColumnDef: + ColumnName Type + { + $$ = []ColumnDef{$1, $2} //TODO 14 + } + +ColumnName: + _IDENTIFIER + { + $$ = $1 //TODO 15 + } + +ColumnNameList: + ColumnName ColumnNameList1 ColumnNameList2 + { + $$ = []ColumnNameList{$1, $2, $3} //TODO 16 + } + +ColumnNameList1: + /* EMPTY */ + { + $$ = []ColumnNameList1(nil) //TODO 17 + } +| ColumnNameList1 ',' ColumnName + { + $$ = append($1.([]ColumnNameList1), ",", $3) //TODO 18 + } + +ColumnNameList2: + /* EMPTY */ + { + $$ = nil //TODO 19 + } +| ',' + { + $$ = "," //TODO 20 + } + +CommitStmt: + _COMMIT + { + $$ = "COMMIT" //TODO 21 + } + +Conversion: + Type '(' Expression ')' + { + $$ = []Conversion{$1, "(", $3, ")"} //TODO 22 + } + +CreateIndexStmt: + _CREATE CreateIndexStmt1 _INDEX CreateIndexStmt2 IndexName _ON TableName '(' CreateIndexStmt3 ')' + { + $$ = []CreateIndexStmt{"CREATE", $2, "INDEX", $4, $5, "ON", $7, "(", $9, ")"} //TODO 23 + } + +CreateIndexStmt1: + /* EMPTY */ + { + $$ = nil //TODO 24 + } +| _UNIQUE + { + $$ = "UNIQUE" //TODO 25 + } + +CreateIndexStmt2: + /* EMPTY */ + { + $$ = nil //TODO 26 + } +| _IF _NOT _EXISTS + { + $$ = []CreateIndexStmt2{"IF", "NOT", "EXISTS"} //TODO 27 + } + +CreateIndexStmt3: + ColumnName + { + $$ = $1 //TODO 28 + } +| _ID Call + { + $$ = []CreateIndexStmt3{"id", $2} //TODO 29 + } + +CreateTableStmt: + _CREATE _TABLE CreateTableStmt1 TableName '(' ColumnDef CreateTableStmt2 CreateTableStmt3 ')' + { + $$ = []CreateTableStmt{"CREATE", "TABLE", $3, $4, "(", $6, $7, $8, ")"} //TODO 30 + } + +CreateTableStmt1: + /* EMPTY */ + { + $$ = nil //TODO 31 + } +| _IF _NOT _EXISTS + { + $$ = []CreateTableStmt1{"IF", "NOT", "EXISTS"} //TODO 32 + } + +CreateTableStmt2: + /* EMPTY */ + { + $$ = []CreateTableStmt2(nil) //TODO 33 + } +| CreateTableStmt2 ',' ColumnDef + { + $$ = append($1.([]CreateTableStmt2), ",", $3) //TODO 34 + } + +CreateTableStmt3: + /* EMPTY */ + { + $$ = nil //TODO 35 + } +| ',' + { + $$ = "," //TODO 36 + } + +DeleteFromStmt: + _DELETE _FROM TableName DeleteFromStmt1 + { + $$ = []DeleteFromStmt{"DELETE", "FROM", $3, $4} //TODO 37 + } + +DeleteFromStmt1: + /* EMPTY */ + { + $$ = nil //TODO 38 + } +| WhereClause + { + $$ = $1 //TODO 39 + } + +DropIndexStmt: + _DROP _INDEX DropIndexStmt1 IndexName + { + $$ = []DropIndexStmt{"DROP", "INDEX", $3, $4} //TODO 40 + } + +DropIndexStmt1: + /* EMPTY */ + { + $$ = nil //TODO 41 + } +| _IF _EXISTS + { + $$ = []DropIndexStmt1{"IF", "EXISTS"} //TODO 42 + } + +DropTableStmt: + _DROP _TABLE DropTableStmt1 TableName + { + $$ = []DropTableStmt{"DROP", "TABLE", $3, $4} //TODO 43 + } + +DropTableStmt1: + /* EMPTY */ + { + $$ = nil //TODO 44 + } +| _IF _EXISTS + { + $$ = []DropTableStmt1{"IF", "EXISTS"} //TODO 45 + } + +EmptyStmt: + /* EMPTY */ + { + $$ = nil //TODO 46 + } + +Expression: + Term Expression1 + { + $$ = []Expression{$1, $2} //TODO 47 + } + +Expression1: + /* EMPTY */ + { + $$ = []Expression1(nil) //TODO 48 + } +| Expression1 Expression11 Term + { + $$ = append($1.([]Expression1), $2, $3) //TODO 49 + } + +Expression11: + _OROR + { + $$ = $1 //TODO 50 + } +| _OR + { + $$ = "OR" //TODO 51 + } + +ExpressionList: + Expression ExpressionList1 ExpressionList2 + { + $$ = []ExpressionList{$1, $2, $3} //TODO 52 + } + +ExpressionList1: + /* EMPTY */ + { + $$ = []ExpressionList1(nil) //TODO 53 + } +| ExpressionList1 ',' Expression + { + $$ = append($1.([]ExpressionList1), ",", $3) //TODO 54 + } + +ExpressionList2: + /* EMPTY */ + { + $$ = nil //TODO 55 + } +| ',' + { + $$ = "," //TODO 56 + } + +Factor: + PrimaryFactor Factor1 Factor2 + { + $$ = []Factor{$1, $2, $3} //TODO 57 + } + +Factor1: + /* EMPTY */ + { + $$ = []Factor1(nil) //TODO 58 + } +| Factor1 Factor11 PrimaryFactor + { + $$ = append($1.([]Factor1), $2, $3) //TODO 59 + } + +Factor11: + _GE + { + $$ = $1 //TODO 60 + } +| '>' + { + $$ = ">" //TODO 61 + } +| _LE + { + $$ = $1 //TODO 62 + } +| '<' + { + $$ = "<" //TODO 63 + } +| _NEQ + { + $$ = $1 //TODO 64 + } +| _EQ + { + $$ = $1 //TODO 65 + } +| _LIKE + { + $$ = "LIKE" //TODO 66 + } + +Factor2: + /* EMPTY */ + { + $$ = nil //TODO 67 + } +| Predicate + { + $$ = $1 //TODO 68 + } + +Field: + Expression Field1 + { + $$ = []Field{$1, $2} //TODO 69 + } + +Field1: + /* EMPTY */ + { + $$ = nil //TODO 70 + } +| _AS _IDENTIFIER + { + $$ = []Field1{"AS", $2} //TODO 71 + } + +FieldList: + Field FieldList1 FieldList2 + { + $$ = []FieldList{$1, $2, $3} //TODO 72 + } + +FieldList1: + /* EMPTY */ + { + $$ = []FieldList1(nil) //TODO 73 + } +| FieldList1 ',' Field + { + $$ = append($1.([]FieldList1), ",", $3) //TODO 74 + } + +FieldList2: + /* EMPTY */ + { + $$ = nil //TODO 75 + } +| ',' + { + $$ = "," //TODO 76 + } + +GroupByClause: + _GROUPBY ColumnNameList + { + $$ = []GroupByClause{"GROUP BY", $2} //TODO 77 + } + +Index: + '[' Expression ']' + { + $$ = []Index{"[", $2, "]"} //TODO 78 + } + +IndexName: + _IDENTIFIER + { + $$ = $1 //TODO 79 + } + +InsertIntoStmt: + _INSERT _INTO TableName InsertIntoStmt1 InsertIntoStmt2 + { + $$ = []InsertIntoStmt{"INSERT", "INTO", $3, $4, $5} //TODO 80 + } + +InsertIntoStmt1: + /* EMPTY */ + { + $$ = nil //TODO 81 + } +| '(' ColumnNameList ')' + { + $$ = []InsertIntoStmt1{"(", $2, ")"} //TODO 82 + } + +InsertIntoStmt2: + Values + { + $$ = $1 //TODO 83 + } +| SelectStmt + { + $$ = $1 //TODO 84 + } + +Limit: + _LIMIT Expression + { + $$ = []Limit{"Limit", $2} //TODO 85 + } + +Literal: + _FALSE + { + $$ = "FALSE" //TODO 86 + } +| _NULL + { + $$ = "NULL" //TODO 87 + } +| _TRUE + { + $$ = "TRUE" //TODO 88 + } +| _FLOAT_LIT + { + $$ = $1 //TODO 89 + } +| _IMAGINARY_LIT + { + $$ = $1 //TODO 90 + } +| _INT_LIT + { + $$ = $1 //TODO 91 + } +| _RUNE_LIT + { + $$ = $1 //TODO 92 + } +| _STRING_LIT + { + $$ = $1 //TODO 93 + } +| _QL_PARAMETER + { + $$ = $1 //TODO 94 + } + +Offset: + _OFFSET Expression + { + $$ = []Offset{"OFFSET", $2} //TODO 95 + } + +Operand: + Literal + { + $$ = $1 //TODO 96 + } +| QualifiedIdent + { + $$ = $1 //TODO 97 + } +| '(' Expression ')' + { + $$ = []Operand{"(", $2, ")"} //TODO 98 + } + +OrderBy: + _ORDER _BY ExpressionList OrderBy1 + { + $$ = []OrderBy{"ORDER", "BY", $3, $4} //TODO 99 + } + +OrderBy1: + /* EMPTY */ + { + $$ = nil //TODO 100 + } +| OrderBy11 + { + $$ = $1 //TODO 101 + } + +OrderBy11: + _ASC + { + $$ = "ASC" //TODO 102 + } +| _DESC + { + $$ = "DESC" //TODO 103 + } + +Predicate: + Predicate1 + { + $$ = $1 //TODO 104 + } + +Predicate1: + Predicate11 Predicate12 + { + $$ = []Predicate1{$1, $2} //TODO 105 + } +| _IS Predicate13 _NULL + { + $$ = []Predicate1{"IS", $2, "NULL"} //TODO 106 + } + +Predicate11: + /* EMPTY */ + { + $$ = nil //TODO 107 + } +| _NOT + { + $$ = "NOT" //TODO 108 + } + +Predicate12: + _IN '(' ExpressionList ')' + { + $$ = []Predicate12{"IN", "(", $3, ")"} //TODO 109 + } +| _IN '(' SelectStmt ')' + { + $$ = []Predicate12{"IN", "(", $3, ")"} //TODO 110 + } +| _BETWEEN PrimaryFactor _AND PrimaryFactor + { + $$ = []Predicate12{"BETWEEN", $2, "AND", $4} //TODO 111 + } + +Predicate13: + /* EMPTY */ + { + $$ = nil //TODO 112 + } +| _NOT + { + $$ = "NOT" //TODO 113 + } + +PrimaryExpression: + Operand + { + $$ = $1 //TODO 114 + } +| Conversion + { + $$ = $1 //TODO 115 + } +| PrimaryExpression Index + { + $$ = []PrimaryExpression{$1, $2} //TODO 116 + } +| PrimaryExpression Slice + { + $$ = []PrimaryExpression{$1, $2} //TODO 117 + } +| PrimaryExpression Call + { + $$ = []PrimaryExpression{$1, $2} //TODO 118 + } + +PrimaryFactor: + PrimaryTerm PrimaryFactor1 + { + $$ = []PrimaryFactor{$1, $2} //TODO 119 + } + +PrimaryFactor1: + /* EMPTY */ + { + $$ = []PrimaryFactor1(nil) //TODO 120 + } +| PrimaryFactor1 PrimaryFactor11 PrimaryTerm + { + $$ = append($1.([]PrimaryFactor1), $2, $3) //TODO 121 + } + +PrimaryFactor11: + '^' + { + $$ = "^" //TODO 122 + } +| '|' + { + $$ = "|" //TODO 123 + } +| '-' + { + $$ = "-" //TODO 124 + } +| '+' + { + $$ = "+" //TODO 125 + } + +PrimaryTerm: + UnaryExpr PrimaryTerm1 + { + $$ = []PrimaryTerm{$1, $2} //TODO 126 + } + +PrimaryTerm1: + /* EMPTY */ + { + $$ = []PrimaryTerm1(nil) //TODO 127 + } +| PrimaryTerm1 PrimaryTerm11 UnaryExpr + { + $$ = append($1.([]PrimaryTerm1), $2, $3) //TODO 128 + } + +PrimaryTerm11: + _ANDNOT + { + $$ = $1 //TODO 129 + } +| '&' + { + $$ = "&" //TODO 130 + } +| _LSH + { + $$ = $1 //TODO 131 + } +| _RSH + { + $$ = $1 //TODO 132 + } +| '%' + { + $$ = "%" //TODO 133 + } +| '/' + { + $$ = "/" //TODO 134 + } +| '*' + { + $$ = "*" //TODO 135 + } + +QualifiedIdent: + _IDENTIFIER QualifiedIdent1 + { + $$ = []QualifiedIdent{$1, $2} //TODO 136 + } + +QualifiedIdent1: + /* EMPTY */ + { + $$ = nil //TODO 137 + } +| '.' _IDENTIFIER + { + $$ = []QualifiedIdent1{".", $2} //TODO 138 + } + +RecordSet: + RecordSet1 RecordSet2 + { + $$ = []RecordSet{$1, $2} //TODO 139 + } + +RecordSet1: + TableName + { + $$ = $1 //TODO 140 + } +| '(' SelectStmt RecordSet11 ')' + { + $$ = []RecordSet1{"(", $2, $3, ")"} //TODO 141 + } + +RecordSet11: + /* EMPTY */ + { + $$ = nil //TODO 142 + } +| ';' + { + $$ = ";" //TODO 143 + } + +RecordSet2: + /* EMPTY */ + { + $$ = nil //TODO 144 + } +| _AS _IDENTIFIER + { + $$ = []RecordSet2{"AS", $2} //TODO 145 + } + +RecordSetList: + RecordSet RecordSetList1 RecordSetList2 + { + $$ = []RecordSetList{$1, $2, $3} //TODO 146 + } + +RecordSetList1: + /* EMPTY */ + { + $$ = []RecordSetList1(nil) //TODO 147 + } +| RecordSetList1 ',' RecordSet + { + $$ = append($1.([]RecordSetList1), ",", $3) //TODO 148 + } + +RecordSetList2: + /* EMPTY */ + { + $$ = nil //TODO 149 + } +| ',' + { + $$ = "," //TODO 150 + } + +RollbackStmt: + _ROLLBACK + { + $$ = "ROLLBACK" //TODO 151 + } + +SelectStmt: + _SELECT SelectStmt1 SelectStmt2 _FROM RecordSetList SelectStmt3 SelectStmt4 SelectStmt5 SelectStmt6 SelectStmt7 + { + $$ = []SelectStmt{"SELECT", $2, $3, "FROM", $5, $6, $7, $8, $9, $10} //TODO 152 + } + +SelectStmt1: + /* EMPTY */ + { + $$ = nil //TODO 153 + } +| _DISTINCT + { + $$ = "DISTINCT" //TODO 154 + } + +SelectStmt2: + '*' + { + $$ = "*" //TODO 155 + } +| FieldList + { + $$ = $1 //TODO 156 + } + +SelectStmt3: + /* EMPTY */ + { + $$ = nil //TODO 157 + } +| WhereClause + { + $$ = $1 //TODO 158 + } + +SelectStmt4: + /* EMPTY */ + { + $$ = nil //TODO 159 + } +| GroupByClause + { + $$ = $1 //TODO 160 + } + +SelectStmt5: + /* EMPTY */ + { + $$ = nil //TODO 161 + } +| OrderBy + { + $$ = $1 //TODO 162 + } + +SelectStmt6: + /* EMPTY */ + { + $$ = nil //TODO 163 + } +| Limit + { + $$ = $1 //TODO 164 + } + +SelectStmt7: + /* EMPTY */ + { + $$ = nil //TODO 165 + } +| Offset + { + $$ = $1 //TODO 166 + } + +Slice: + '[' Slice1 ':' Slice2 ']' + { + $$ = []Slice{"[", $2, ":", $4, "]"} //TODO 167 + } + +Slice1: + /* EMPTY */ + { + $$ = nil //TODO 168 + } +| Expression + { + $$ = $1 //TODO 169 + } + +Slice2: + /* EMPTY */ + { + $$ = nil //TODO 170 + } +| Expression + { + $$ = $1 //TODO 171 + } + +Start: + StatementList + { + _parserResult = $1 //TODO 172 + } + +Statement: + EmptyStmt + { + $$ = $1 //TODO 173 + } +| AlterTableStmt + { + $$ = $1 //TODO 174 + } +| BeginTransactionStmt + { + $$ = $1 //TODO 175 + } +| CommitStmt + { + $$ = $1 //TODO 176 + } +| CreateIndexStmt + { + $$ = $1 //TODO 177 + } +| CreateTableStmt + { + $$ = $1 //TODO 178 + } +| DeleteFromStmt + { + $$ = $1 //TODO 179 + } +| DropIndexStmt + { + $$ = $1 //TODO 180 + } +| DropTableStmt + { + $$ = $1 //TODO 181 + } +| InsertIntoStmt + { + $$ = $1 //TODO 182 + } +| RollbackStmt + { + $$ = $1 //TODO 183 + } +| SelectStmt + { + $$ = $1 //TODO 184 + } +| TruncateTableStmt + { + $$ = $1 //TODO 185 + } +| UpdateStmt + { + $$ = $1 //TODO 186 + } + +StatementList: + Statement StatementList1 + { + $$ = []StatementList{$1, $2} //TODO 187 + } + +StatementList1: + /* EMPTY */ + { + $$ = []StatementList1(nil) //TODO 188 + } +| StatementList1 ';' Statement + { + $$ = append($1.([]StatementList1), ";", $3) //TODO 189 + } + +TableName: + _IDENTIFIER + { + $$ = $1 //TODO 190 + } + +Term: + Factor Term1 + { + $$ = []Term{$1, $2} //TODO 191 + } + +Term1: + /* EMPTY */ + { + $$ = []Term1(nil) //TODO 192 + } +| Term1 Term11 Factor + { + $$ = append($1.([]Term1), $2, $3) //TODO 193 + } + +Term11: + _ANDAND + { + $$ = $1 //TODO 194 + } +| _AND + { + $$ = "AND" //TODO 195 + } + +TruncateTableStmt: + _TRUNCATE _TABLE TableName + { + $$ = []TruncateTableStmt{"TRUNCATE", "TABLE", $3} //TODO 196 + } + +Type: + _BIGINT + { + $$ = "bigint" //TODO 197 + } +| _BIGRAT + { + $$ = "bigrat" //TODO 198 + } +| _BLOB + { + $$ = "blob" //TODO 199 + } +| _BOOL + { + $$ = "bool" //TODO 200 + } +| _BYTE + { + $$ = "byte" //TODO 201 + } +| _COMPLEX128 + { + $$ = "complex128" //TODO 202 + } +| _COMPLEX64 + { + $$ = "complex64" //TODO 203 + } +| _DURATION + { + $$ = "duration" //TODO 204 + } +| _FLOAT + { + $$ = "float" //TODO 205 + } +| _FLOAT32 + { + $$ = "float32" //TODO 206 + } +| _FLOAT64 + { + $$ = "float64" //TODO 207 + } +| _INT + { + $$ = "int" //TODO 208 + } +| _INT16 + { + $$ = "int16" //TODO 209 + } +| _INT32 + { + $$ = "int32" //TODO 210 + } +| _INT64 + { + $$ = "int64" //TODO 211 + } +| _INT8 + { + $$ = "int8" //TODO 212 + } +| _RUNE + { + $$ = "rune" //TODO 213 + } +| _STRING + { + $$ = "string" //TODO 214 + } +| _TIME + { + $$ = "time" //TODO 215 + } +| _UINT + { + $$ = "uint" //TODO 216 + } +| _UINT16 + { + $$ = "uint16" //TODO 217 + } +| _UINT32 + { + $$ = "uint32" //TODO 218 + } +| _UINT64 + { + $$ = "uint64" //TODO 219 + } +| _UINT8 + { + $$ = "uint8" //TODO 220 + } + +UnaryExpr: + UnaryExpr1 PrimaryExpression + { + $$ = []UnaryExpr{$1, $2} //TODO 221 + } + +UnaryExpr1: + /* EMPTY */ + { + $$ = nil //TODO 222 + } +| UnaryExpr11 + { + $$ = $1 //TODO 223 + } + +UnaryExpr11: + '^' + { + $$ = "^" //TODO 224 + } +| '!' + { + $$ = "!" //TODO 225 + } +| '-' + { + $$ = "-" //TODO 226 + } +| '+' + { + $$ = "+" //TODO 227 + } + +UpdateStmt: + _UPDATE TableName UpdateStmt1 AssignmentList UpdateStmt2 + { + $$ = []UpdateStmt{"UPDATE", $2, $3, $4, $5} //TODO 228 + } + +UpdateStmt1: + /* EMPTY */ + { + $$ = nil //TODO 229 + } +| _SET + { + $$ = "SET" //TODO 230 + } + +UpdateStmt2: + /* EMPTY */ + { + $$ = nil //TODO 231 + } +| WhereClause + { + $$ = $1 //TODO 232 + } + +Values: + _VALUES '(' ExpressionList ')' Values1 Values2 + { + $$ = []Values{"VALUES", "(", $3, ")", $5, $6} //TODO 233 + } + +Values1: + /* EMPTY */ + { + $$ = []Values1(nil) //TODO 234 + } +| Values1 ',' '(' ExpressionList ')' + { + $$ = append($1.([]Values1), ",", "(", $4, ")") //TODO 235 + } + +Values2: + /* EMPTY */ + { + $$ = nil //TODO 236 + } +| ',' + { + $$ = "," //TODO 237 + } + +WhereClause: + _WHERE Expression + { + $$ = []WhereClause{"WHERE", $2} //TODO 238 + } + +%% + +//TODO remove demo stuff below + +var _parserResult interface{} + +type ( + AlterTableStmt interface{} + AlterTableStmt1 interface{} + Assignment interface{} + AssignmentList interface{} + AssignmentList1 interface{} + AssignmentList2 interface{} + BeginTransactionStmt interface{} + Call interface{} + Call1 interface{} + ColumnDef interface{} + ColumnName interface{} + ColumnNameList interface{} + ColumnNameList1 interface{} + ColumnNameList2 interface{} + CommitStmt interface{} + Conversion interface{} + CreateIndexStmt interface{} + CreateIndexStmt1 interface{} + CreateIndexStmt2 interface{} + CreateIndexStmt3 interface{} + CreateTableStmt interface{} + CreateTableStmt1 interface{} + CreateTableStmt2 interface{} + CreateTableStmt3 interface{} + DeleteFromStmt interface{} + DeleteFromStmt1 interface{} + DropIndexStmt interface{} + DropIndexStmt1 interface{} + DropTableStmt interface{} + DropTableStmt1 interface{} + EmptyStmt interface{} + Expression interface{} + Expression1 interface{} + Expression11 interface{} + ExpressionList interface{} + ExpressionList1 interface{} + ExpressionList2 interface{} + Factor interface{} + Factor1 interface{} + Factor11 interface{} + Factor2 interface{} + Field interface{} + Field1 interface{} + FieldList interface{} + FieldList1 interface{} + FieldList2 interface{} + GroupByClause interface{} + Index interface{} + IndexName interface{} + InsertIntoStmt interface{} + InsertIntoStmt1 interface{} + InsertIntoStmt2 interface{} + Limit interface{} + Literal interface{} + Offset interface{} + Operand interface{} + OrderBy interface{} + OrderBy1 interface{} + OrderBy11 interface{} + Predicate interface{} + Predicate1 interface{} + Predicate11 interface{} + Predicate12 interface{} + Predicate13 interface{} + PrimaryExpression interface{} + PrimaryFactor interface{} + PrimaryFactor1 interface{} + PrimaryFactor11 interface{} + PrimaryTerm interface{} + PrimaryTerm1 interface{} + PrimaryTerm11 interface{} + QualifiedIdent interface{} + QualifiedIdent1 interface{} + RecordSet interface{} + RecordSet1 interface{} + RecordSet11 interface{} + RecordSet2 interface{} + RecordSetList interface{} + RecordSetList1 interface{} + RecordSetList2 interface{} + RollbackStmt interface{} + SelectStmt interface{} + SelectStmt1 interface{} + SelectStmt2 interface{} + SelectStmt3 interface{} + SelectStmt4 interface{} + SelectStmt5 interface{} + SelectStmt6 interface{} + SelectStmt7 interface{} + Slice interface{} + Slice1 interface{} + Slice2 interface{} + Start interface{} + Statement interface{} + StatementList interface{} + StatementList1 interface{} + TableName interface{} + Term interface{} + Term1 interface{} + Term11 interface{} + TruncateTableStmt interface{} + Type interface{} + UnaryExpr interface{} + UnaryExpr1 interface{} + UnaryExpr11 interface{} + UpdateStmt interface{} + UpdateStmt1 interface{} + UpdateStmt2 interface{} + Values interface{} + Values1 interface{} + Values2 interface{} + WhereClause interface{} +) + +func _dump() { + s := fmt.Sprintf("%#v", _parserResult) + s = strings.Replace(s, "%", "%%", -1) + s = strings.Replace(s, "{", "{%i\n", -1) + s = strings.Replace(s, "}", "%u\n}", -1) + s = strings.Replace(s, ", ", ",\n", -1) + var buf bytes.Buffer + strutil.IndentFormatter(&buf, ". ").Format(s) + buf.WriteString("\n") + a := strings.Split(buf.String(), "\n") + for _, v := range a { + if strings.HasSuffix(v, "(nil)") || strings.HasSuffix(v, "(nil),") { + continue + } + + fmt.Println(v) + } +} + +// End of demo stuff diff --git a/Godeps/_workspace/src/github.com/cznic/ql/ql/Makefile b/Godeps/_workspace/src/github.com/cznic/ql/ql/Makefile new file mode 100644 index 00000000000..e795538a760 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/ql/Makefile @@ -0,0 +1,24 @@ +# Copyright 2014 The ql Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +.PHONY: all clean nuke + +all: editor + go vet + go install + make todo + +clean: + go clean + rm -f *~ ql + +editor: + go fmt + go install + +todo: + @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go || true + @grep -n TODO *.go || true + @grep -n BUG *.go || true + @grep -n println *.go || true diff --git a/Godeps/_workspace/src/github.com/cznic/ql/ql/README.md b/Godeps/_workspace/src/github.com/cznic/ql/ql/README.md new file mode 100644 index 00000000000..07139fd617d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/ql/README.md @@ -0,0 +1,10 @@ +ql +== + +Command ql is a utility to explore a database, prototype a schema or test drive a query, etc. + +Installation + + $ go get github.com/cznic/ql/ql + +Documentation: [godoc.org/github.com/cznic/ql/ql](http://godoc.org/github.com/cznic/ql/ql) diff --git a/Godeps/_workspace/src/github.com/cznic/ql/ql/main.go b/Godeps/_workspace/src/github.com/cznic/ql/ql/main.go new file mode 100644 index 00000000000..74c449ad64e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/ql/main.go @@ -0,0 +1,200 @@ +// Copyright 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Command ql is a utility to explore a database, prototype a schema or test +// drive a query, etc. +// +// Installation: +// +// $ go get github.com/cznic/ql/ql +// +// Usage: +// +// ql [-db name] [-schema regexp] [-tables regexp] [-fld] statement_list +// +// Options: +// +// -db name Name of the database to use. Defaults to "ql.db". +// If the DB file does not exists it is created automatically. +// +// -schema re If re != "" show the CREATE statements of matching tables and exit. +// +// -tables re If re != "" show the matching table names and exit. +// +// -fld First row of a query result set will show field names. +// +// statement_list QL statements to execute. +// If no non flag arguments are present, ql reads from stdin. +// The list is wrapped into an automatic transaction. +// +// Example: +// +// $ ql 'create table t (i int, s string)' +// $ ql << EOF +// > insert into t values +// > (1, "a"), +// > (2, "b"), +// > (3, "c"), +// > EOF +// $ ql 'select * from t' +// 3, "c" +// 2, "b" +// 1, "a" +// $ ql -fld 'select * from t where i != 2 order by s' +// "i", "s" +// 1, "a" +// 3, "c" +// $ +package main + +import ( + "bufio" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "regexp" + "sort" + "strings" + + "github.com/cznic/ql" +) + +func str(data []interface{}) string { + a := make([]string, len(data)) + for i, v := range data { + switch x := v.(type) { + case string: + a[i] = fmt.Sprintf("%q", x) + default: + a[i] = fmt.Sprint(x) + } + } + return strings.Join(a, ", ") +} + +func main() { + if err := do(); err != nil { + log.Fatal(err) + } +} + +func do() (err error) { + oDB := flag.String("db", "ql.db", "The DB file to open. It'll be created if missing.") + oFlds := flag.Bool("fld", false, "Show recordset's field names.") + oSchema := flag.String("schema", "", "If non empty, show the CREATE statements of matching tables and exit.") + oTables := flag.String("tables", "", "If non empty, list matching table names and exit.") + flag.Parse() + + db, err := ql.OpenFile(*oDB, &ql.Options{CanCreate: true}) + if err != nil { + return err + } + + defer func() { + ec := db.Close() + switch { + case ec != nil && err != nil: + log.Println(ec) + case ec != nil: + err = ec + } + }() + + if pat := *oSchema; pat != "" { + re, err := regexp.Compile(pat) + if err != nil { + return err + } + + nfo, err := db.Info() + if err != nil { + return err + } + + r := []string{} + for _, ti := range nfo.Tables { + if !re.MatchString(ti.Name) { + continue + } + + a := []string{} + for _, ci := range ti.Columns { + a = append(a, fmt.Sprintf("%s %s", ci.Name, ci.Type)) + } + r = append(r, fmt.Sprintf("CREATE TABLE %s (%s);", ti.Name, strings.Join(a, ", "))) + } + sort.Strings(r) + if len(r) != 0 { + fmt.Println(strings.Join(r, "\n")) + } + return nil + } + + if pat := *oTables; pat != "" { + re, err := regexp.Compile(pat) + if err != nil { + return err + } + + nfo, err := db.Info() + if err != nil { + return err + } + + r := []string{} + for _, ti := range nfo.Tables { + if !re.MatchString(ti.Name) { + continue + } + + r = append(r, ti.Name) + } + sort.Strings(r) + if len(r) != 0 { + fmt.Println(strings.Join(r, "\n")) + } + return nil + } + + var src string + switch n := flag.NArg(); n { + case 0: + b, err := ioutil.ReadAll(bufio.NewReader(os.Stdin)) + if err != nil { + return err + } + + src = string(b) + default: + a := make([]string, n) + for i := range a { + a[i] = flag.Arg(i) + } + src = strings.Join(a, " ") + } + + src = "BEGIN TRANSACTION; " + src + "; COMMIT;" + l, err := ql.Compile(src) + if err != nil { + log.Println(src) + return err + } + + rs, i, err := db.Execute(ql.NewRWCtx(), l) + if err != nil { + a := strings.Split(strings.TrimSpace(fmt.Sprint(l)), "\n") + return fmt.Errorf("%v: %s", err, a[i]) + } + + if len(rs) == 0 { + return + } + + return rs[len(rs)-1].Do(*oFlds, func(data []interface{}) (bool, error) { + fmt.Println(str(data)) + return true, nil + }) +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/scanner.go b/Godeps/_workspace/src/github.com/cznic/ql/scanner.go new file mode 100644 index 00000000000..4e6332c787d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/scanner.go @@ -0,0 +1,3788 @@ +/* + +Copyright (c) 2014 The ql Authors. All rights reserved. +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file. + +CAUTION: If this file is 'scanner.go', it was generated +automatically from 'scanner.l' - DO NOT EDIT in that case! + +*/ + +package ql + +import ( + "fmt" + "math" + "strconv" + "unicode" +) + +type lexer struct { + agg []bool + c int + col int + errs []error + i int + lcol int + line int + list []stmt + ncol int + nline int + params int + sc int + src string + val []byte +} + +func newLexer(src string) (l *lexer) { + l = &lexer{ + src: src, + nline: 1, + ncol: 0, + } + l.next() + return +} + +func (l *lexer) next() int { + if l.c != 0 { + l.val = append(l.val, byte(l.c)) + } + l.c = 0 + if l.i < len(l.src) { + l.c = int(l.src[l.i]) + l.i++ + } + switch l.c { + case '\n': + l.lcol = l.ncol + l.nline++ + l.ncol = 0 + default: + l.ncol++ + } + return l.c +} + +func (l *lexer) err0(ln, c int, s string, arg ...interface{}) { + err := fmt.Errorf(fmt.Sprintf("%d:%d ", ln, c)+s, arg...) + l.errs = append(l.errs, err) +} + +func (l *lexer) err(s string, arg ...interface{}) { + l.err0(l.line, l.col, s, arg...) +} + +func (l *lexer) Error(s string) { + l.err(s) +} + +func (l *lexer) Lex(lval *yySymType) (r int) { + //defer func() { dbg("Lex -> %d(%#x)", r, r) }() + defer func() { + lval.line, lval.col = l.line, l.col + }() + const ( + INITIAL = iota + S1 + S2 + ) + + c0, c := 0, l.c + +yystate0: + + l.val = l.val[:0] + c0, l.line, l.col = l.c, l.nline, l.ncol + + switch yyt := l.sc; yyt { + default: + panic(fmt.Errorf(`invalid start condition %d`, yyt)) + case 0: // start condition: INITIAL + goto yystart1 + case 1: // start condition: S1 + goto yystart291 + case 2: // start condition: S2 + goto yystart296 + } + + goto yystate0 // silence unused label error + goto yystate1 // silence unused label error +yystate1: + c = l.next() +yystart1: + switch { + default: + goto yystate3 // c >= '\x01' && c <= '\b' || c == '\v' || c == '\f' || c >= '\x0e' && c <= '\x1f' || c == '#' || c == '%%' || c >= '(' && c <= ',' || c == ':' || c == ';' || c == '@' || c >= '[' && c <= '^' || c == '{' || c >= '}' && c <= 'ÿ' + case c == '!': + goto yystate6 + case c == '"': + goto yystate8 + case c == '$' || c == '?': + goto yystate9 + case c == '&': + goto yystate11 + case c == '-': + goto yystate19 + case c == '.': + goto yystate21 + case c == '/': + goto yystate27 + case c == '0': + goto yystate32 + case c == '<': + goto yystate40 + case c == '=': + goto yystate43 + case c == '>': + goto yystate45 + case c == 'A' || c == 'a': + goto yystate48 + case c == 'B' || c == 'b': + goto yystate60 + case c == 'C' || c == 'c': + goto yystate87 + case c == 'D' || c == 'd': + goto yystate111 + case c == 'E' || c == 'e': + goto yystate136 + case c == 'F' || c == 'f': + goto yystate142 + case c == 'G' || c == 'g': + goto yystate158 + case c == 'H' || c == 'J' || c == 'K' || c == 'M' || c == 'P' || c == 'Q' || c >= 'X' && c <= 'Z' || c == '_' || c == 'h' || c == 'j' || c == 'k' || c == 'm' || c == 'p' || c == 'q' || c >= 'x' && c <= 'z': + goto yystate163 + case c == 'I' || c == 'i': + goto yystate164 + case c == 'L' || c == 'l': + goto yystate184 + case c == 'N' || c == 'n': + goto yystate191 + case c == 'O' || c == 'o': + goto yystate197 + case c == 'R' || c == 'r': + goto yystate208 + case c == 'S' || c == 's': + goto yystate219 + case c == 'T' || c == 't': + goto yystate231 + case c == 'U' || c == 'u': + goto yystate256 + case c == 'V' || c == 'v': + goto yystate277 + case c == 'W' || c == 'w': + goto yystate283 + case c == '\'': + goto yystate14 + case c == '\n': + goto yystate5 + case c == '\t' || c == '\r' || c == ' ': + goto yystate4 + case c == '\x00': + goto yystate2 + case c == '`': + goto yystate288 + case c == '|': + goto yystate289 + case c >= '1' && c <= '9': + goto yystate38 + } + +yystate2: + c = l.next() + goto yyrule1 + +yystate3: + c = l.next() + goto yyrule94 + +yystate4: + c = l.next() + switch { + default: + goto yyrule2 + case c == '\t' || c == '\n' || c == '\r' || c == ' ': + goto yystate5 + } + +yystate5: + c = l.next() + switch { + default: + goto yyrule2 + case c == '\t' || c == '\n' || c == '\r' || c == ' ': + goto yystate5 + } + +yystate6: + c = l.next() + switch { + default: + goto yyrule94 + case c == '=': + goto yystate7 + } + +yystate7: + c = l.next() + goto yyrule21 + +yystate8: + c = l.next() + goto yyrule10 + +yystate9: + c = l.next() + switch { + default: + goto yyrule94 + case c >= '0' && c <= '9': + goto yystate10 + } + +yystate10: + c = l.next() + switch { + default: + goto yyrule93 + case c >= '0' && c <= '9': + goto yystate10 + } + +yystate11: + c = l.next() + switch { + default: + goto yyrule94 + case c == '&': + goto yystate12 + case c == '^': + goto yystate13 + } + +yystate12: + c = l.next() + goto yyrule15 + +yystate13: + c = l.next() + goto yyrule16 + +yystate14: + c = l.next() + switch { + default: + goto yyrule94 + case c == '\'': + goto yystate16 + case c == '\\': + goto yystate17 + case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate15 + } + +yystate15: + c = l.next() + switch { + default: + goto yyabort + case c == '\'': + goto yystate16 + case c == '\\': + goto yystate17 + case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate15 + } + +yystate16: + c = l.next() + goto yyrule12 + +yystate17: + c = l.next() + switch { + default: + goto yyabort + case c == '\'': + goto yystate18 + case c == '\\': + goto yystate17 + case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate15 + } + +yystate18: + c = l.next() + switch { + default: + goto yyrule12 + case c == '\'': + goto yystate16 + case c == '\\': + goto yystate17 + case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate15 + } + +yystate19: + c = l.next() + switch { + default: + goto yyrule94 + case c == '-': + goto yystate20 + } + +yystate20: + c = l.next() + switch { + default: + goto yyrule3 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': + goto yystate20 + } + +yystate21: + c = l.next() + switch { + default: + goto yyrule94 + case c >= '0' && c <= '9': + goto yystate22 + } + +yystate22: + c = l.next() + switch { + default: + goto yyrule9 + case c == 'E' || c == 'e': + goto yystate23 + case c == 'i': + goto yystate26 + case c >= '0' && c <= '9': + goto yystate22 + } + +yystate23: + c = l.next() + switch { + default: + goto yyabort + case c == '+' || c == '-': + goto yystate24 + case c >= '0' && c <= '9': + goto yystate25 + } + +yystate24: + c = l.next() + switch { + default: + goto yyabort + case c >= '0' && c <= '9': + goto yystate25 + } + +yystate25: + c = l.next() + switch { + default: + goto yyrule9 + case c == 'i': + goto yystate26 + case c >= '0' && c <= '9': + goto yystate25 + } + +yystate26: + c = l.next() + goto yyrule7 + +yystate27: + c = l.next() + switch { + default: + goto yyrule94 + case c == '*': + goto yystate28 + case c == '/': + goto yystate31 + } + +yystate28: + c = l.next() + switch { + default: + goto yyabort + case c == '*': + goto yystate29 + case c >= '\x01' && c <= ')' || c >= '+' && c <= 'ÿ': + goto yystate28 + } + +yystate29: + c = l.next() + switch { + default: + goto yyabort + case c == '*': + goto yystate29 + case c == '/': + goto yystate30 + case c >= '\x01' && c <= ')' || c >= '+' && c <= '.' || c >= '0' && c <= 'ÿ': + goto yystate28 + } + +yystate30: + c = l.next() + goto yyrule5 + +yystate31: + c = l.next() + switch { + default: + goto yyrule4 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': + goto yystate31 + } + +yystate32: + c = l.next() + switch { + default: + goto yyrule8 + case c == '.': + goto yystate22 + case c == '8' || c == '9': + goto yystate34 + case c == 'E' || c == 'e': + goto yystate23 + case c == 'X' || c == 'x': + goto yystate36 + case c == 'i': + goto yystate35 + case c >= '0' && c <= '7': + goto yystate33 + } + +yystate33: + c = l.next() + switch { + default: + goto yyrule8 + case c == '.': + goto yystate22 + case c == '8' || c == '9': + goto yystate34 + case c == 'E' || c == 'e': + goto yystate23 + case c == 'i': + goto yystate35 + case c >= '0' && c <= '7': + goto yystate33 + } + +yystate34: + c = l.next() + switch { + default: + goto yyabort + case c == '.': + goto yystate22 + case c == 'E' || c == 'e': + goto yystate23 + case c == 'i': + goto yystate35 + case c >= '0' && c <= '9': + goto yystate34 + } + +yystate35: + c = l.next() + goto yyrule6 + +yystate36: + c = l.next() + switch { + default: + goto yyabort + case c >= '0' && c <= '9' || c >= 'A' && c <= 'F' || c >= 'a' && c <= 'f': + goto yystate37 + } + +yystate37: + c = l.next() + switch { + default: + goto yyrule8 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'F' || c >= 'a' && c <= 'f': + goto yystate37 + } + +yystate38: + c = l.next() + switch { + default: + goto yyrule8 + case c == '.': + goto yystate22 + case c == 'E' || c == 'e': + goto yystate23 + case c == 'i': + goto yystate35 + case c >= '0' && c <= '9': + goto yystate39 + } + +yystate39: + c = l.next() + switch { + default: + goto yyrule8 + case c == '.': + goto yystate22 + case c == 'E' || c == 'e': + goto yystate23 + case c == 'i': + goto yystate35 + case c >= '0' && c <= '9': + goto yystate39 + } + +yystate40: + c = l.next() + switch { + default: + goto yyrule94 + case c == '<': + goto yystate41 + case c == '=': + goto yystate42 + } + +yystate41: + c = l.next() + goto yyrule17 + +yystate42: + c = l.next() + goto yyrule18 + +yystate43: + c = l.next() + switch { + default: + goto yyrule94 + case c == '=': + goto yystate44 + } + +yystate44: + c = l.next() + goto yyrule19 + +yystate45: + c = l.next() + switch { + default: + goto yyrule94 + case c == '=': + goto yystate46 + case c == '>': + goto yystate47 + } + +yystate46: + c = l.next() + goto yyrule20 + +yystate47: + c = l.next() + goto yyrule23 + +yystate48: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'D' || c == 'd': + goto yystate50 + case c == 'L' || c == 'l': + goto yystate52 + case c == 'N' || c == 'n': + goto yystate56 + case c == 'S' || c == 's': + goto yystate58 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'C' || c >= 'E' && c <= 'K' || c == 'M' || c >= 'O' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'c' || c >= 'e' && c <= 'k' || c == 'm' || c >= 'o' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate49: + c = l.next() + switch { + default: + goto yyrule92 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate50: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'D' || c == 'd': + goto yystate51 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'C' || c >= 'E' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'c' || c >= 'e' && c <= 'z': + goto yystate49 + } + +yystate51: + c = l.next() + switch { + default: + goto yyrule24 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate52: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate53 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate53: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate54 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate54: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'R' || c == 'r': + goto yystate55 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate55: + c = l.next() + switch { + default: + goto yyrule25 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate56: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'D' || c == 'd': + goto yystate57 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'C' || c >= 'E' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'c' || c >= 'e' && c <= 'z': + goto yystate49 + } + +yystate57: + c = l.next() + switch { + default: + goto yyrule26 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate58: + c = l.next() + switch { + default: + goto yyrule28 + case c == 'C' || c == 'c': + goto yystate59 + case c >= '0' && c <= '9' || c == 'A' || c == 'B' || c >= 'D' && c <= 'Z' || c == '_' || c == 'a' || c == 'b' || c >= 'd' && c <= 'z': + goto yystate49 + } + +yystate59: + c = l.next() + switch { + default: + goto yyrule27 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate60: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate61 + case c == 'I' || c == 'i': + goto yystate70 + case c == 'L' || c == 'l': + goto yystate78 + case c == 'O' || c == 'o': + goto yystate81 + case c == 'Y' || c == 'y': + goto yystate84 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'H' || c == 'J' || c == 'K' || c == 'M' || c == 'N' || c >= 'P' && c <= 'X' || c == 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'h' || c == 'j' || c == 'k' || c == 'm' || c == 'n' || c >= 'p' && c <= 'x' || c == 'z': + goto yystate49 + } + +yystate61: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'G' || c == 'g': + goto yystate62 + case c == 'T' || c == 't': + goto yystate65 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'F' || c >= 'H' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'f' || c >= 'h' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate62: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate63 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate63: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate64 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate64: + c = l.next() + switch { + default: + goto yyrule29 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate65: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'W' || c == 'w': + goto yystate66 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'V' || c >= 'X' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'v' || c >= 'x' && c <= 'z': + goto yystate49 + } + +yystate66: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate67 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate67: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate68 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate68: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate69 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate69: + c = l.next() + switch { + default: + goto yyrule30 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate70: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'G' || c == 'g': + goto yystate71 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'F' || c >= 'H' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'f' || c >= 'h' && c <= 'z': + goto yystate49 + } + +yystate71: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate72 + case c == 'R' || c == 'r': + goto yystate75 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate72: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate73 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate73: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate74 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate74: + c = l.next() + switch { + default: + goto yyrule68 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate75: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate76 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate76: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate77 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate77: + c = l.next() + switch { + default: + goto yyrule69 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate78: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate79 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate79: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'B' || c == 'b': + goto yystate80 + case c >= '0' && c <= '9' || c == 'A' || c >= 'C' && c <= 'Z' || c == '_' || c == 'a' || c >= 'c' && c <= 'z': + goto yystate49 + } + +yystate80: + c = l.next() + switch { + default: + goto yyrule70 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate81: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate82 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate82: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate83 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate83: + c = l.next() + switch { + default: + goto yyrule71 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate84: + c = l.next() + switch { + default: + goto yyrule31 + case c == 'T' || c == 't': + goto yystate85 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate85: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate86 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate86: + c = l.next() + switch { + default: + goto yyrule72 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate87: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate88 + case c == 'R' || c == 'r': + goto yystate106 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c == 'P' || c == 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c == 'p' || c == 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate88: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate89 + case c == 'M' || c == 'm': + goto yystate93 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'N' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'n' && c <= 'z': + goto yystate49 + } + +yystate89: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'U' || c == 'u': + goto yystate90 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'a' && c <= 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate90: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'M' || c == 'm': + goto yystate91 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'L' || c >= 'N' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'l' || c >= 'n' && c <= 'z': + goto yystate49 + } + +yystate91: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate92 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate92: + c = l.next() + switch { + default: + goto yyrule32 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate93: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'M' || c == 'm': + goto yystate94 + case c == 'P' || c == 'p': + goto yystate97 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'L' || c == 'N' || c == 'O' || c >= 'Q' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'l' || c == 'n' || c == 'o' || c >= 'q' && c <= 'z': + goto yystate49 + } + +yystate94: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate95 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate95: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate96 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate96: + c = l.next() + switch { + default: + goto yyrule33 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate97: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate98 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate98: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate99 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate99: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'X' || c == 'x': + goto yystate100 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'W' || c == 'Y' || c == 'Z' || c == '_' || c >= 'a' && c <= 'w' || c == 'y' || c == 'z': + goto yystate49 + } + +yystate100: + c = l.next() + switch { + default: + goto yyrule92 + case c == '0' || c >= '2' && c <= '5' || c >= '7' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + case c == '1': + goto yystate101 + case c == '6': + goto yystate104 + } + +yystate101: + c = l.next() + switch { + default: + goto yyrule92 + case c == '0' || c == '1' || c >= '3' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + case c == '2': + goto yystate102 + } + +yystate102: + c = l.next() + switch { + default: + goto yyrule92 + case c == '8': + goto yystate103 + case c >= '0' && c <= '7' || c == '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate103: + c = l.next() + switch { + default: + goto yyrule73 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate104: + c = l.next() + switch { + default: + goto yyrule92 + case c == '4': + goto yystate105 + case c >= '0' && c <= '3' || c >= '5' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate105: + c = l.next() + switch { + default: + goto yyrule74 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate106: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate107 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate107: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate108 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate108: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate109 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate109: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate110 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate110: + c = l.next() + switch { + default: + goto yyrule34 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate111: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate112 + case c == 'I' || c == 'i': + goto yystate119 + case c == 'R' || c == 'r': + goto yystate126 + case c == 'U' || c == 'u': + goto yystate129 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'H' || c >= 'J' && c <= 'Q' || c == 'S' || c == 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'h' || c >= 'j' && c <= 'q' || c == 's' || c == 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate112: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate113 + case c == 'S' || c == 's': + goto yystate117 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate113: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate114 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate114: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate115 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate115: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate116 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate116: + c = l.next() + switch { + default: + goto yyrule35 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate117: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'C' || c == 'c': + goto yystate118 + case c >= '0' && c <= '9' || c == 'A' || c == 'B' || c >= 'D' && c <= 'Z' || c == '_' || c == 'a' || c == 'b' || c >= 'd' && c <= 'z': + goto yystate49 + } + +yystate118: + c = l.next() + switch { + default: + goto yyrule36 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate119: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'S' || c == 's': + goto yystate120 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate120: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate121 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate121: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate122 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate122: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate123 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate123: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'C' || c == 'c': + goto yystate124 + case c >= '0' && c <= '9' || c == 'A' || c == 'B' || c >= 'D' && c <= 'Z' || c == '_' || c == 'a' || c == 'b' || c >= 'd' && c <= 'z': + goto yystate49 + } + +yystate124: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate125 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate125: + c = l.next() + switch { + default: + goto yyrule37 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate126: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate127 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate127: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'P' || c == 'p': + goto yystate128 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'O' || c >= 'Q' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'o' || c >= 'q' && c <= 'z': + goto yystate49 + } + +yystate128: + c = l.next() + switch { + default: + goto yyrule38 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate129: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'R' || c == 'r': + goto yystate130 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate130: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate131 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate131: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate132 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate132: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate133 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate133: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate134 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate134: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate135 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate135: + c = l.next() + switch { + default: + goto yyrule75 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate136: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'X' || c == 'x': + goto yystate137 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'W' || c == 'Y' || c == 'Z' || c == '_' || c >= 'a' && c <= 'w' || c == 'y' || c == 'z': + goto yystate49 + } + +yystate137: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate138 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate138: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'S' || c == 's': + goto yystate139 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate139: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate140 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate140: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'S' || c == 's': + goto yystate141 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate141: + c = l.next() + switch { + default: + goto yyrule39 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate142: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate143 + case c == 'L' || c == 'l': + goto yystate147 + case c == 'R' || c == 'r': + goto yystate155 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'K' || c >= 'M' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'k' || c >= 'm' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate143: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate144 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate144: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'S' || c == 's': + goto yystate145 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate145: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate146 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate146: + c = l.next() + switch { + default: + goto yyrule66 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate147: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate148 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate148: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate149 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate149: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate150 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate150: + c = l.next() + switch { + default: + goto yyrule76 + case c == '3': + goto yystate151 + case c == '6': + goto yystate153 + case c >= '0' && c <= '2' || c == '4' || c == '5' || c >= '7' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate151: + c = l.next() + switch { + default: + goto yyrule92 + case c == '0' || c == '1' || c >= '3' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + case c == '2': + goto yystate152 + } + +yystate152: + c = l.next() + switch { + default: + goto yyrule77 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate153: + c = l.next() + switch { + default: + goto yyrule92 + case c == '4': + goto yystate154 + case c >= '0' && c <= '3' || c >= '5' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate154: + c = l.next() + switch { + default: + goto yyrule78 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate155: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate156 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate156: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'M' || c == 'm': + goto yystate157 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'L' || c >= 'N' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'l' || c >= 'n' && c <= 'z': + goto yystate49 + } + +yystate157: + c = l.next() + switch { + default: + goto yyrule40 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate158: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'R' || c == 'r': + goto yystate159 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate159: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate160 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate160: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'U' || c == 'u': + goto yystate161 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'a' && c <= 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate161: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'P' || c == 'p': + goto yystate162 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'O' || c >= 'Q' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'o' || c >= 'q' && c <= 'z': + goto yystate49 + } + +yystate162: + c = l.next() + switch { + default: + goto yyrule41 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate163: + c = l.next() + switch { + default: + goto yyrule92 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate164: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'F' || c == 'f': + goto yystate165 + case c == 'N' || c == 'n': + goto yystate166 + case c == 'S' || c == 's': + goto yystate183 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'E' || c >= 'G' && c <= 'M' || c >= 'O' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'e' || c >= 'g' && c <= 'm' || c >= 'o' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate165: + c = l.next() + switch { + default: + goto yyrule42 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate166: + c = l.next() + switch { + default: + goto yyrule46 + case c == 'D' || c == 'd': + goto yystate167 + case c == 'S' || c == 's': + goto yystate170 + case c == 'T' || c == 't': + goto yystate174 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'C' || c >= 'E' && c <= 'R' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'c' || c >= 'e' && c <= 'r' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate167: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate168 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate168: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'X' || c == 'x': + goto yystate169 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'W' || c == 'Y' || c == 'Z' || c == '_' || c >= 'a' && c <= 'w' || c == 'y' || c == 'z': + goto yystate49 + } + +yystate169: + c = l.next() + switch { + default: + goto yyrule43 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate170: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate171 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate171: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'R' || c == 'r': + goto yystate172 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate172: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate173 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate173: + c = l.next() + switch { + default: + goto yyrule44 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate174: + c = l.next() + switch { + default: + goto yyrule79 + case c == '0' || c == '2' || c == '4' || c == '5' || c == '7' || c == '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + case c == '1': + goto yystate175 + case c == '3': + goto yystate177 + case c == '6': + goto yystate179 + case c == '8': + goto yystate181 + case c == 'O' || c == 'o': + goto yystate182 + } + +yystate175: + c = l.next() + switch { + default: + goto yyrule92 + case c == '6': + goto yystate176 + case c >= '0' && c <= '5' || c >= '7' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate176: + c = l.next() + switch { + default: + goto yyrule80 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate177: + c = l.next() + switch { + default: + goto yyrule92 + case c == '0' || c == '1' || c >= '3' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + case c == '2': + goto yystate178 + } + +yystate178: + c = l.next() + switch { + default: + goto yyrule81 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate179: + c = l.next() + switch { + default: + goto yyrule92 + case c == '4': + goto yystate180 + case c >= '0' && c <= '3' || c >= '5' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate180: + c = l.next() + switch { + default: + goto yyrule82 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate181: + c = l.next() + switch { + default: + goto yyrule83 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate182: + c = l.next() + switch { + default: + goto yyrule45 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate183: + c = l.next() + switch { + default: + goto yyrule47 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate184: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate185 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate185: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'K' || c == 'k': + goto yystate186 + case c == 'M' || c == 'm': + goto yystate188 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'J' || c == 'L' || c >= 'N' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'j' || c == 'l' || c >= 'n' && c <= 'z': + goto yystate49 + } + +yystate186: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate187 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate187: + c = l.next() + switch { + default: + goto yyrule48 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate188: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate189 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate189: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate190 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate190: + c = l.next() + switch { + default: + goto yyrule49 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate191: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate192 + case c == 'U' || c == 'u': + goto yystate194 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate192: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate193 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate193: + c = l.next() + switch { + default: + goto yyrule50 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate194: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate195 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate195: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate196 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate196: + c = l.next() + switch { + default: + goto yyrule65 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate197: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'F' || c == 'f': + goto yystate198 + case c == 'N' || c == 'n': + goto yystate203 + case c == 'R' || c == 'r': + goto yystate204 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'E' || c >= 'G' && c <= 'M' || c >= 'O' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'e' || c >= 'g' && c <= 'm' || c >= 'o' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate198: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'F' || c == 'f': + goto yystate199 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'E' || c >= 'G' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'e' || c >= 'g' && c <= 'z': + goto yystate49 + } + +yystate199: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'S' || c == 's': + goto yystate200 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate200: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate201 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate201: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate202 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate202: + c = l.next() + switch { + default: + goto yyrule51 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate203: + c = l.next() + switch { + default: + goto yyrule52 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate204: + c = l.next() + switch { + default: + goto yyrule53 + case c == 'D' || c == 'd': + goto yystate205 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'C' || c >= 'E' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'c' || c >= 'e' && c <= 'z': + goto yystate49 + } + +yystate205: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate206 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate206: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'R' || c == 'r': + goto yystate207 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate207: + c = l.next() + switch { + default: + goto yyrule54 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate208: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate209 + case c == 'U' || c == 'u': + goto yystate216 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate209: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate210 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate210: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate211 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate211: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'B' || c == 'b': + goto yystate212 + case c >= '0' && c <= '9' || c == 'A' || c >= 'C' && c <= 'Z' || c == '_' || c == 'a' || c >= 'c' && c <= 'z': + goto yystate49 + } + +yystate212: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate213 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate213: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'C' || c == 'c': + goto yystate214 + case c >= '0' && c <= '9' || c == 'A' || c == 'B' || c >= 'D' && c <= 'Z' || c == '_' || c == 'a' || c == 'b' || c >= 'd' && c <= 'z': + goto yystate49 + } + +yystate214: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'K' || c == 'k': + goto yystate215 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'J' || c >= 'L' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'j' || c >= 'l' && c <= 'z': + goto yystate49 + } + +yystate215: + c = l.next() + switch { + default: + goto yyrule55 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate216: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate217 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate217: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate218 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate218: + c = l.next() + switch { + default: + goto yyrule84 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate219: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate220 + case c == 'T' || c == 't': + goto yystate226 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate220: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate221 + case c == 'T' || c == 't': + goto yystate225 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate221: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate222 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate222: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'C' || c == 'c': + goto yystate223 + case c >= '0' && c <= '9' || c == 'A' || c == 'B' || c >= 'D' && c <= 'Z' || c == '_' || c == 'a' || c == 'b' || c >= 'd' && c <= 'z': + goto yystate49 + } + +yystate223: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate224 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate224: + c = l.next() + switch { + default: + goto yyrule56 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate225: + c = l.next() + switch { + default: + goto yyrule57 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate226: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'R' || c == 'r': + goto yystate227 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate227: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate228 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate228: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate229 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate229: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'G' || c == 'g': + goto yystate230 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'F' || c >= 'H' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'f' || c >= 'h' && c <= 'z': + goto yystate49 + } + +yystate230: + c = l.next() + switch { + default: + goto yyrule85 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate231: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate232 + case c == 'I' || c == 'i': + goto yystate236 + case c == 'R' || c == 'r': + goto yystate239 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'H' || c >= 'J' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'h' || c >= 'j' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate232: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'B' || c == 'b': + goto yystate233 + case c >= '0' && c <= '9' || c == 'A' || c >= 'C' && c <= 'Z' || c == '_' || c == 'a' || c >= 'c' && c <= 'z': + goto yystate49 + } + +yystate233: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate234 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate234: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate235 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate235: + c = l.next() + switch { + default: + goto yyrule58 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate236: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'M' || c == 'm': + goto yystate237 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'L' || c >= 'N' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'l' || c >= 'n' && c <= 'z': + goto yystate49 + } + +yystate237: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate238 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate238: + c = l.next() + switch { + default: + goto yyrule86 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate239: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate240 + case c == 'U' || c == 'u': + goto yystate249 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'b' && c <= 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate240: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate241 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate241: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'S' || c == 's': + goto yystate242 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate242: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate243 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate243: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'C' || c == 'c': + goto yystate244 + case c >= '0' && c <= '9' || c == 'A' || c == 'B' || c >= 'D' && c <= 'Z' || c == '_' || c == 'a' || c == 'b' || c >= 'd' && c <= 'z': + goto yystate49 + } + +yystate244: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate245 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate245: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate246 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate246: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'O' || c == 'o': + goto yystate247 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'N' || c >= 'P' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'n' || c >= 'p' && c <= 'z': + goto yystate49 + } + +yystate247: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate248 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate248: + c = l.next() + switch { + default: + goto yyrule59 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate249: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate250 + case c == 'N' || c == 'n': + goto yystate251 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate250: + c = l.next() + switch { + default: + goto yyrule67 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate251: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'C' || c == 'c': + goto yystate252 + case c >= '0' && c <= '9' || c == 'A' || c == 'B' || c >= 'D' && c <= 'Z' || c == '_' || c == 'a' || c == 'b' || c >= 'd' && c <= 'z': + goto yystate49 + } + +yystate252: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate253 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate253: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate254 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate254: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate255 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate255: + c = l.next() + switch { + default: + goto yyrule60 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate256: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate257 + case c == 'N' || c == 'n': + goto yystate267 + case c == 'P' || c == 'p': + goto yystate272 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'M' || c == 'O' || c >= 'Q' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'm' || c == 'o' || c >= 'q' && c <= 'z': + goto yystate49 + } + +yystate257: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'N' || c == 'n': + goto yystate258 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'M' || c >= 'O' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'm' || c >= 'o' && c <= 'z': + goto yystate49 + } + +yystate258: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate259 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate259: + c = l.next() + switch { + default: + goto yyrule87 + case c == '0' || c == '2' || c == '4' || c == '5' || c == '7' || c == '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + case c == '1': + goto yystate260 + case c == '3': + goto yystate262 + case c == '6': + goto yystate264 + case c == '8': + goto yystate266 + } + +yystate260: + c = l.next() + switch { + default: + goto yyrule92 + case c == '6': + goto yystate261 + case c >= '0' && c <= '5' || c >= '7' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate261: + c = l.next() + switch { + default: + goto yyrule88 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate262: + c = l.next() + switch { + default: + goto yyrule92 + case c == '0' || c == '1' || c >= '3' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + case c == '2': + goto yystate263 + } + +yystate263: + c = l.next() + switch { + default: + goto yyrule89 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate264: + c = l.next() + switch { + default: + goto yyrule92 + case c == '4': + goto yystate265 + case c >= '0' && c <= '3' || c >= '5' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate265: + c = l.next() + switch { + default: + goto yyrule90 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate266: + c = l.next() + switch { + default: + goto yyrule91 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate267: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'I' || c == 'i': + goto yystate268 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'H' || c >= 'J' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'h' || c >= 'j' && c <= 'z': + goto yystate49 + } + +yystate268: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'Q' || c == 'q': + goto yystate269 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'P' || c >= 'R' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'p' || c >= 'r' && c <= 'z': + goto yystate49 + } + +yystate269: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'U' || c == 'u': + goto yystate270 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'a' && c <= 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate270: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate271 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate271: + c = l.next() + switch { + default: + goto yyrule62 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate272: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'D' || c == 'd': + goto yystate273 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'C' || c >= 'E' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'c' || c >= 'e' && c <= 'z': + goto yystate49 + } + +yystate273: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate274 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate274: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'T' || c == 't': + goto yystate275 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'S' || c >= 'U' && c <= 'Z' || c == '_' || c >= 'a' && c <= 's' || c >= 'u' && c <= 'z': + goto yystate49 + } + +yystate275: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate276 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate276: + c = l.next() + switch { + default: + goto yyrule61 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate277: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'A' || c == 'a': + goto yystate278 + case c >= '0' && c <= '9' || c >= 'B' && c <= 'Z' || c == '_' || c >= 'b' && c <= 'z': + goto yystate49 + } + +yystate278: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'L' || c == 'l': + goto yystate279 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'K' || c >= 'M' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'k' || c >= 'm' && c <= 'z': + goto yystate49 + } + +yystate279: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'U' || c == 'u': + goto yystate280 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'T' || c >= 'V' && c <= 'Z' || c == '_' || c >= 'a' && c <= 't' || c >= 'v' && c <= 'z': + goto yystate49 + } + +yystate280: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate281 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate281: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'S' || c == 's': + goto yystate282 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'R' || c >= 'T' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'r' || c >= 't' && c <= 'z': + goto yystate49 + } + +yystate282: + c = l.next() + switch { + default: + goto yyrule63 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate283: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'H' || c == 'h': + goto yystate284 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'G' || c >= 'I' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'g' || c >= 'i' && c <= 'z': + goto yystate49 + } + +yystate284: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate285 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate285: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'R' || c == 'r': + goto yystate286 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Q' || c >= 'S' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'q' || c >= 's' && c <= 'z': + goto yystate49 + } + +yystate286: + c = l.next() + switch { + default: + goto yyrule92 + case c == 'E' || c == 'e': + goto yystate287 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'D' || c >= 'F' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'd' || c >= 'f' && c <= 'z': + goto yystate49 + } + +yystate287: + c = l.next() + switch { + default: + goto yyrule64 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate49 + } + +yystate288: + c = l.next() + goto yyrule11 + +yystate289: + c = l.next() + switch { + default: + goto yyrule94 + case c == '|': + goto yystate290 + } + +yystate290: + c = l.next() + goto yyrule22 + + goto yystate291 // silence unused label error +yystate291: + c = l.next() +yystart291: + switch { + default: + goto yystate292 // c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ' + case c == '"': + goto yystate293 + case c == '\\': + goto yystate294 + case c == '\x00': + goto yystate2 + } + +yystate292: + c = l.next() + switch { + default: + goto yyabort + case c == '"': + goto yystate293 + case c == '\\': + goto yystate294 + case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate292 + } + +yystate293: + c = l.next() + goto yyrule13 + +yystate294: + c = l.next() + switch { + default: + goto yyabort + case c == '"': + goto yystate295 + case c == '\\': + goto yystate294 + case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate292 + } + +yystate295: + c = l.next() + switch { + default: + goto yyrule13 + case c == '"': + goto yystate293 + case c == '\\': + goto yystate294 + case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate292 + } + + goto yystate296 // silence unused label error +yystate296: + c = l.next() +yystart296: + switch { + default: + goto yystate297 // c >= '\x01' && c <= '_' || c >= 'a' && c <= 'ÿ' + case c == '\x00': + goto yystate2 + case c == '`': + goto yystate298 + } + +yystate297: + c = l.next() + switch { + default: + goto yyabort + case c == '`': + goto yystate298 + case c >= '\x01' && c <= '_' || c >= 'a' && c <= 'ÿ': + goto yystate297 + } + +yystate298: + c = l.next() + goto yyrule14 + +yyrule1: // \0 + { + return 0 + } +yyrule2: // [ \t\n\r]+ + + goto yystate0 +yyrule3: // --.* + + goto yystate0 +yyrule4: // \/\/.* + + goto yystate0 +yyrule5: // \/\*([^*]|\*+[^*/])*\*+\/ + + goto yystate0 +yyrule6: // {imaginary_ilit} + { + return l.int(lval, true) + } +yyrule7: // {imaginary_lit} + { + return l.float(lval, true) + } +yyrule8: // {int_lit} + { + return l.int(lval, false) + } +yyrule9: // {float_lit} + { + return l.float(lval, false) + } +yyrule10: // \" + { + l.sc = S1 + goto yystate0 + } +yyrule11: // ` + { + l.sc = S2 + goto yystate0 + } +yyrule12: // '(\\.|[^'])*' + { + if ret := l.str(lval, ""); ret != stringLit { + return ret + } + lval.item = idealRune(lval.item.(string)[0]) + return intLit + } +yyrule13: // (\\.|[^\"])*\" + { + return l.str(lval, "\"") + } +yyrule14: // ([^`]|\n)*` + { + return l.str(lval, "`") + } +yyrule15: // "&&" + { + return andand + } +yyrule16: // "&^" + { + return andnot + } +yyrule17: // "<<" + { + return lsh + } +yyrule18: // "<=" + { + return le + } +yyrule19: // "==" + { + return eq + } +yyrule20: // ">=" + { + return ge + } +yyrule21: // "!=" + { + return neq + } +yyrule22: // "||" + { + return oror + } +yyrule23: // ">>" + { + return rsh + } +yyrule24: // {add} + { + return add + } +yyrule25: // {alter} + { + return alter + } +yyrule26: // {and} + { + return and + } +yyrule27: // {asc} + { + return asc + } +yyrule28: // {as} + { + return as + } +yyrule29: // {begin} + { + return begin + } +yyrule30: // {between} + { + return between + } +yyrule31: // {by} + { + return by + } +yyrule32: // {column} + { + return column + } +yyrule33: // {commit} + { + return commit + } +yyrule34: // {create} + { + return create + } +yyrule35: // {delete} + { + return deleteKwd + } +yyrule36: // {desc} + { + return desc + } +yyrule37: // {distinct} + { + return distinct + } +yyrule38: // {drop} + { + return drop + } +yyrule39: // {exists} + { + return exists + } +yyrule40: // {from} + { + return from + } +yyrule41: // {group} + { + return group + } +yyrule42: // {if} + { + return ifKwd + } +yyrule43: // {index} + { + return index + } +yyrule44: // {insert} + { + return insert + } +yyrule45: // {into} + { + return into + } +yyrule46: // {in} + { + return in + } +yyrule47: // {is} + { + return is + } +yyrule48: // {like} + { + return like + } +yyrule49: // {limit} + { + return limit + } +yyrule50: // {not} + { + return not + } +yyrule51: // {offset} + { + return offset + } +yyrule52: // {on} + { + return on + } +yyrule53: // {or} + { + return or + } +yyrule54: // {order} + { + return order + } +yyrule55: // {rollback} + { + return rollback + } +yyrule56: // {select} + { + l.agg = append(l.agg, false) + return selectKwd + } +yyrule57: // {set} + { + return set + } +yyrule58: // {table} + { + return tableKwd + } +yyrule59: // {transaction} + { + return transaction + } +yyrule60: // {truncate} + { + return truncate + } +yyrule61: // {update} + { + return update + } +yyrule62: // {unique} + { + return unique + } +yyrule63: // {values} + { + return values + } +yyrule64: // {where} + { + return where + } +yyrule65: // {null} + { + lval.item = nil + return null + } +yyrule66: // {false} + { + lval.item = false + return falseKwd + } +yyrule67: // {true} + { + lval.item = true + return trueKwd + } +yyrule68: // {bigint} + { + lval.item = qBigInt + return bigIntType + } +yyrule69: // {bigrat} + { + lval.item = qBigRat + return bigRatType + } +yyrule70: // {blob} + { + lval.item = qBlob + return blobType + } +yyrule71: // {bool} + { + lval.item = qBool + return boolType + } +yyrule72: // {byte} + { + lval.item = qUint8 + return byteType + } +yyrule73: // {complex}128 + { + lval.item = qComplex128 + return complex128Type + } +yyrule74: // {complex}64 + { + lval.item = qComplex64 + return complex64Type + } +yyrule75: // {duration} + { + lval.item = qDuration + return durationType + } +yyrule76: // {float} + { + lval.item = qFloat64 + return floatType + } +yyrule77: // {float}32 + { + lval.item = qFloat32 + return float32Type + } +yyrule78: // {float}64 + { + lval.item = qFloat64 + return float64Type + } +yyrule79: // {int} + { + lval.item = qInt64 + return intType + } +yyrule80: // {int}16 + { + lval.item = qInt16 + return int16Type + } +yyrule81: // {int}32 + { + lval.item = qInt32 + return int32Type + } +yyrule82: // {int}64 + { + lval.item = qInt64 + return int64Type + } +yyrule83: // {int}8 + { + lval.item = qInt8 + return int8Type + } +yyrule84: // {rune} + { + lval.item = qInt32 + return runeType + } +yyrule85: // {string} + { + lval.item = qString + return stringType + } +yyrule86: // {time} + { + lval.item = qTime + return timeType + } +yyrule87: // {uint} + { + lval.item = qUint64 + return uintType + } +yyrule88: // {uint}16 + { + lval.item = qUint16 + return uint16Type + } +yyrule89: // {uint}32 + { + lval.item = qUint32 + return uint32Type + } +yyrule90: // {uint}64 + { + lval.item = qUint64 + return uint64Type + } +yyrule91: // {uint}8 + { + lval.item = qUint8 + return uint8Type + } +yyrule92: // {ident} + { + lval.item = string(l.val) + return identifier + } +yyrule93: // ($|\?){D} + { + lval.item, _ = strconv.Atoi(string(l.val[1:])) + return qlParam + } +yyrule94: // . + { + return c0 + } + panic("unreachable") + + goto yyabort // silence unused label error + +yyabort: // no lexem recognized + return int(unicode.ReplacementChar) +} + +func (l *lexer) npos() (line, col int) { + if line, col = l.nline, l.ncol; col == 0 { + line-- + col = l.lcol + 1 + } + return +} + +func (l *lexer) str(lval *yySymType, pref string) int { + l.sc = 0 + s := pref + string(l.val) + s, err := strconv.Unquote(s) + if err != nil { + l.err("string literal: %v", err) + return int(unicode.ReplacementChar) + } + + lval.item = s + return stringLit +} + +func (l *lexer) int(lval *yySymType, im bool) int { + if im { + l.val = l.val[:len(l.val)-1] + } + n, err := strconv.ParseUint(string(l.val), 0, 64) + if err != nil { + l.err("integer literal: %v", err) + return int(unicode.ReplacementChar) + } + + if im { + lval.item = idealComplex(complex(0, float64(n))) + return imaginaryLit + } + + switch { + case n < math.MaxInt64: + lval.item = idealInt(n) + default: + lval.item = idealUint(n) + } + return intLit +} + +func (l *lexer) float(lval *yySymType, im bool) int { + if im { + l.val = l.val[:len(l.val)-1] + } + n, err := strconv.ParseFloat(string(l.val), 64) + if err != nil { + l.err("float literal: %v", err) + return int(unicode.ReplacementChar) + } + + if im { + lval.item = idealComplex(complex(0, n)) + return imaginaryLit + } + + lval.item = idealFloat(n) + return floatLit +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/scanner.l b/Godeps/_workspace/src/github.com/cznic/ql/scanner.l new file mode 100644 index 00000000000..43abbcdeab1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/scanner.l @@ -0,0 +1,444 @@ +/* + +Copyright (c) 2014 The ql Authors. All rights reserved. +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file. + +CAUTION: If this file is 'scanner.go', it was generated +automatically from 'scanner.l' - DO NOT EDIT in that case! + +*/ + +%{ + +package ql + +import ( + "fmt" + "math" + "strconv" + "unicode" +) + +type lexer struct { + agg []bool + c int + col int + errs []error + i int + lcol int + line int + list []stmt + ncol int + nline int + params int + sc int + src string + val []byte +} + +func newLexer(src string) (l *lexer) { + l = &lexer{ + src: src, + nline: 1, + ncol: 0, + } + l.next() + return +} + +func (l *lexer) next() int { + if l.c != 0 { + l.val = append(l.val, byte(l.c)) + } + l.c = 0 + if l.i < len(l.src) { + l.c = int(l.src[l.i]) + l.i++ + } + switch l.c { + case '\n': + l.lcol = l.ncol + l.nline++ + l.ncol = 0 + default: + l.ncol++ + } + return l.c +} + +func (l *lexer) err0(ln, c int, s string, arg ...interface{}) { + err := fmt.Errorf(fmt.Sprintf("%d:%d ", ln, c)+s, arg...) + l.errs = append(l.errs, err) +} + +func (l *lexer) err(s string, arg ...interface{}) { + l.err0(l.line, l.col, s, arg...) +} + +func (l *lexer) Error(s string) { + l.err(s) +} + +func (l *lexer) Lex(lval *yySymType) (r int) { + //defer func() { dbg("Lex -> %d(%#x)", r, r) }() + defer func() { + lval.line, lval.col = l.line, l.col + }() + const ( + INITIAL = iota + S1 + S2 + ) + + c0, c := 0, l.c +%} + +int_lit {decimal_lit}|{octal_lit}|{hex_lit} +decimal_lit [1-9][0-9]* +octal_lit 0[0-7]* +hex_lit 0[xX][0-9a-fA-F]+ + +float_lit {D}"."{D}?{E}?|{D}{E}|"."{D}{E}? +D [0-9]+ +E [eE][-+]?[0-9]+ + +imaginary_ilit {D}i +imaginary_lit {float_lit}i + +a [aA] +b [bB] +c [cC] +d [dD] +e [eE] +f [fF] +g [gG] +h [hH] +i [iI] +j [jJ] +k [kK] +l [lL] +m [mM] +n [nN] +o [oO] +p [pP] +q [qQ] +r [rR] +s [sS] +t [tT] +u [uU] +v [vV] +w [wW] +x [xX] +y [yY] +z [zZ] + +add {a}{d}{d} +alter {a}{l}{t}{e}{r} +and {a}{n}{d} +as {a}{s} +asc {a}{s}{c} +begin {b}{e}{g}{i}{n} +between {b}{e}{t}{w}{e}{e}{n} +by {b}{y} +column {c}{o}{l}{u}{m}{n} +commit {c}{o}{m}{m}{i}{t} +create {c}{r}{e}{a}{t}{e} +delete {d}{e}{l}{e}{t}{e} +desc {d}{e}{s}{c} +distinct {d}{i}{s}{t}{i}{n}{c}{t} +drop {d}{r}{o}{p} +exists {e}{x}{i}{s}{t}{s} +from {f}{r}{o}{m} +group {g}{r}{o}{u}{p} +if {i}{f} +in {i}{n} +index {i}{n}{d}{e}{x} +insert {i}{n}{s}{e}{r}{t} +into {i}{n}{t}{o} +is {i}{s} +like {l}{i}{k}{e} +limit {l}{i}{m}{i}{t} +not {n}{o}{t} +offset {o}{f}{f}{s}{e}{t} +on {o}{n} +or {o}{r} +order {o}{r}{d}{e}{r} +rollback {r}{o}{l}{l}{b}{a}{c}{k} +select {s}{e}{l}{e}{c}{t} +set {s}{e}{t} +table {t}{a}{b}{l}{e} +transaction {t}{r}{a}{n}{s}{a}{c}{t}{i}{o}{n} +truncate {t}{r}{u}{n}{c}{a}{t}{e} +unique {u}{n}{i}{q}{u}{e} +update {u}{p}{d}{a}{t}{e} +values {v}{a}{l}{u}{e}{s} +where {w}{h}{e}{r}{e} + +null {n}{u}{l}{l} +false {f}{a}{l}{s}{e} +true {t}{r}{u}{e} + +bigint {b}{i}{g}{i}{n}{t} +bigrat {b}{i}{g}{r}{a}{t} +blob {b}{l}{o}{b} +bool {b}{o}{o}{l} +byte {b}{y}{t}{e} +complex {c}{o}{m}{p}{l}{e}{x} +duration {d}{u}{r}{a}{t}{i}{o}{n} +float {f}{l}{o}{a}{t} +int {i}{n}{t} +rune {r}{u}{n}{e} +string {s}{t}{r}{i}{n}{g} +time {t}{i}{m}{e} +uint {u}{i}{n}{t} + +idchar0 [a-zA-Z_] +idchars {idchar0}|[0-9] +ident {idchar0}{idchars}* + +%yyc c +%yyn c = l.next() +%yyt l.sc + +%x S1 S2 + +%% + l.val = l.val[:0] + c0, l.line, l.col = l.c, l.nline, l.ncol + +<*>\0 return 0 + +[ \t\n\r]+ +--.* +\/\/.* +\/\*([^*]|\*+[^*/])*\*+\/ + +{imaginary_ilit} return l.int(lval, true) +{imaginary_lit} return l.float(lval, true) +{int_lit} return l.int(lval, false) +{float_lit} return l.float(lval, false) + +\" l.sc = S1 +` l.sc = S2 + +'(\\.|[^'])*' if ret := l.str(lval, ""); ret != stringLit { + return ret + } + lval.item = idealRune(lval.item.(string)[0]) + return intLit + +(\\.|[^\"])*\" return l.str(lval, "\"") +([^`]|\n)*` return l.str(lval, "`") + +"&&" return andand +"&^" return andnot +"<<" return lsh +"<=" return le +"==" return eq +">=" return ge +"!=" return neq +"||" return oror +">>" return rsh + +{add} return add +{alter} return alter +{and} return and +{asc} return asc +{as} return as +{begin} return begin +{between} return between +{by} return by +{column} return column +{commit} return commit +{create} return create +{delete} return deleteKwd +{desc} return desc +{distinct} return distinct +{drop} return drop +{exists} return exists +{from} return from +{group} return group +{if} return ifKwd +{index} return index +{insert} return insert +{into} return into +{in} return in +{is} return is +{like} return like +{limit} return limit +{not} return not +{offset} return offset +{on} return on +{or} return or +{order} return order +{rollback} return rollback + +{select} l.agg = append(l.agg, false) + return selectKwd + +{set} return set +{table} return tableKwd +{transaction} return transaction +{truncate} return truncate +{update} return update +{unique} return unique +{values} return values +{where} return where + +{null} lval.item = nil + return null + +{false} lval.item = false + return falseKwd + +{true} lval.item = true + return trueKwd + +{bigint} lval.item = qBigInt + return bigIntType + +{bigrat} lval.item = qBigRat + return bigRatType + +{blob} lval.item = qBlob + return blobType + +{bool} lval.item = qBool + return boolType + +{byte} lval.item = qUint8 + return byteType + +{complex}128 lval.item = qComplex128 + return complex128Type + +{complex}64 lval.item = qComplex64 + return complex64Type + +{duration} lval.item = qDuration + return durationType + +{float} lval.item = qFloat64 + return floatType + +{float}32 lval.item = qFloat32 + return float32Type + +{float}64 lval.item = qFloat64 + return float64Type + +{int} lval.item = qInt64 + return intType + +{int}16 lval.item = qInt16 + return int16Type + +{int}32 lval.item = qInt32 + return int32Type + +{int}64 lval.item = qInt64 + return int64Type + +{int}8 lval.item = qInt8 + return int8Type + +{rune} lval.item = qInt32 + return runeType + +{string} lval.item = qString + return stringType + +{time} lval.item = qTime + return timeType + +{uint} lval.item = qUint64 + return uintType + +{uint}16 lval.item = qUint16 + return uint16Type + +{uint}32 lval.item = qUint32 + return uint32Type + +{uint}64 lval.item = qUint64 + return uint64Type + +{uint}8 lval.item = qUint8 + return uint8Type + +{ident} lval.item = string(l.val) + return identifier + +($|\?){D} lval.item, _ = strconv.Atoi(string(l.val[1:])) + return qlParam + +. return c0 + +%% + return int(unicode.ReplacementChar) +} + +func (l *lexer) npos() (line, col int) { + if line, col = l.nline, l.ncol; col == 0 { + line-- + col = l.lcol+1 + } + return +} + +func (l *lexer) str(lval *yySymType, pref string) int { + l.sc = 0 + s := pref + string(l.val) + s, err := strconv.Unquote(s) + if err != nil { + l.err("string literal: %v", err) + return int(unicode.ReplacementChar) + } + + lval.item = s + return stringLit +} + +func (l *lexer) int(lval *yySymType, im bool) int { + if im { + l.val = l.val[:len(l.val)-1] + } + n, err := strconv.ParseUint(string(l.val), 0, 64) + if err != nil { + l.err("integer literal: %v", err) + return int(unicode.ReplacementChar) + } + + if im { + lval.item = idealComplex(complex(0, float64(n))) + return imaginaryLit + } + + switch { + case n < math.MaxInt64: + lval.item = idealInt(n) + default: + lval.item = idealUint(n) + } + return intLit +} + +func (l *lexer) float(lval *yySymType, im bool) int { + if im { + l.val = l.val[:len(l.val)-1] + } + n, err := strconv.ParseFloat(string(l.val), 64) + if err != nil { + l.err("float literal: %v", err) + return int(unicode.ReplacementChar) + } + + if im { + lval.item = idealComplex(complex(0, n)) + return imaginaryLit + } + + lval.item = idealFloat(n) + return floatLit +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/scanner_test.go b/Godeps/_workspace/src/github.com/cznic/ql/scanner_test.go new file mode 100644 index 00000000000..150c902a6ce --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/scanner_test.go @@ -0,0 +1,86 @@ +// Copyright (c) 2014 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "fmt" + "testing" + "unicode" +) + +var bad = int(unicode.ReplacementChar) + +func tok2name(i int) string { + if i == unicode.ReplacementChar { + return "" + } + + if i < 128 { + return fmt.Sprintf("tok-'%c'", i) + } + + return fmt.Sprintf("tok-%d", i) +} + +func TestScaner0(t *testing.T) { + table := []struct { + src string + tok, line, col, nline, ncol int + val string + }{ + {"a", identifier, 1, 1, 1, 2, "a"}, + {" a", identifier, 1, 2, 1, 3, "a"}, + {"a ", identifier, 1, 1, 1, 2, "a"}, + {" a ", identifier, 1, 2, 1, 3, "a"}, + {"\na", identifier, 2, 1, 2, 2, "a"}, + + {"a\n", identifier, 1, 1, 1, 2, "a"}, + {"\na\n", identifier, 2, 1, 2, 2, "a"}, + {"\n a", identifier, 2, 2, 2, 3, "a"}, + {"a \n", identifier, 1, 1, 1, 2, "a"}, + {"\n a \n", identifier, 2, 2, 2, 3, "a"}, + + {"ab", identifier, 1, 1, 1, 3, "ab"}, + {" ab", identifier, 1, 2, 1, 4, "ab"}, + {"ab ", identifier, 1, 1, 1, 3, "ab"}, + {" ab ", identifier, 1, 2, 1, 4, "ab"}, + {"\nab", identifier, 2, 1, 2, 3, "ab"}, + + {"ab\n", identifier, 1, 1, 1, 3, "ab"}, + {"\nab\n", identifier, 2, 1, 2, 3, "ab"}, + {"\n ab", identifier, 2, 2, 2, 4, "ab"}, + {"ab \n", identifier, 1, 1, 1, 3, "ab"}, + {"\n ab \n", identifier, 2, 2, 2, 4, "ab"}, + + {"c", identifier, 1, 1, 1, 2, "c"}, + {"cR", identifier, 1, 1, 1, 3, "cR"}, + {"cRe", identifier, 1, 1, 1, 4, "cRe"}, + {"cReA", identifier, 1, 1, 1, 5, "cReA"}, + {"cReAt", identifier, 1, 1, 1, 6, "cReAt"}, + + {"cReATe", create, 1, 1, 1, 7, "cReATe"}, + {"cReATeD", identifier, 1, 1, 1, 8, "cReATeD"}, + {"2", intLit, 1, 1, 1, 2, "2"}, + {"2.", floatLit, 1, 1, 1, 3, "2."}, + {"2.3", floatLit, 1, 1, 1, 4, "2.3"}, + } + + lval := &yySymType{} + for i, test := range table { + l := newLexer(test.src) + tok := l.Lex(lval) + nline, ncol := l.npos() + val := string(l.val) + if tok != test.tok || l.line != test.line || l.col != test.col || + nline != test.nline || ncol != test.ncol || + val != test.val { + t.Fatalf( + "%d g: %s %d:%d-%d:%d %q, e: %s %d:%d-%d:%d %q", + i, tok2name(tok), l.line, l.col, nline, ncol, val, + tok2name(test.tok), test.line, test.col, test.nline, test.ncol, test.val, + ) + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/stmt.go b/Godeps/_workspace/src/github.com/cznic/ql/stmt.go new file mode 100644 index 00000000000..455b3be68f0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/stmt.go @@ -0,0 +1,900 @@ +// Copyright (c) 2014 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "fmt" + "log" + "strings" + "sync" +) + +// NOTE: all stmt implementations must be safe for concurrent use by multiple +// goroutines. If the exec method requires any execution domain local data, +// they must be held out of the implementing instance. +var ( + _ stmt = (*alterTableAddStmt)(nil) + _ stmt = (*alterTableDropColumnStmt)(nil) + _ stmt = (*createIndexStmt)(nil) + _ stmt = (*createTableStmt)(nil) + _ stmt = (*deleteStmt)(nil) //TODO use indices (need dlist) + _ stmt = (*dropIndexStmt)(nil) + _ stmt = (*dropTableStmt)(nil) + _ stmt = (*insertIntoStmt)(nil) + _ stmt = (*selectStmt)(nil) + _ stmt = (*truncateTableStmt)(nil) + _ stmt = (*updateStmt)(nil) //TODO use indices + _ stmt = beginTransactionStmt{} + _ stmt = commitStmt{} + _ stmt = rollbackStmt{} +) + +type stmt interface { + // never invoked for + // - beginTransactionStmt + // - commitStmt + // - rollbackStmt + exec(ctx *execCtx) (Recordset, error) + + // return value ignored for + // - beginTransactionStmt + // - commitStmt + // - rollbackStmt + isUpdating() bool + String() string +} + +type execCtx struct { //LATER +shared temp + db *DB + arg []interface{} +} + +type updateStmt struct { + tableName string + list []assignment + where expression +} + +func (s *updateStmt) String() string { + u := fmt.Sprintf("UPDATE TABLE %s", s.tableName) + a := make([]string, len(s.list)) + for i, v := range s.list { + a[i] = v.String() + } + w := "" + if s.where != nil { + w = fmt.Sprintf(" WHERE %s", s.where) + } + return fmt.Sprintf("%s %s%s", u, strings.Join(a, ", "), w) +} + +func (s *updateStmt) exec(ctx *execCtx) (_ Recordset, err error) { + t, ok := ctx.db.root.tables[s.tableName] + if !ok { + return nil, fmt.Errorf("UPDATE: table %s does not exist", s.tableName) + } + + tcols := make([]*col, len(s.list)) + for i, asgn := range s.list { + col := findCol(t.cols, asgn.colName) + if col == nil { + return nil, fmt.Errorf("UPDATE: unknown column %s", asgn.colName) + } + tcols[i] = col + } + + m := map[interface{}]interface{}{} + var nh int64 + expr := s.where + blobCols := t.blobCols() + cc := ctx.db.cc + var old []interface{} + var touched []bool + if t.hasIndices() { + old = make([]interface{}, len(t.cols0)) + touched = make([]bool, len(t.cols0)) + } + for h := t.head; h != 0; h = nh { + // Read can return lazily expanded chunks + data, err := t.store.Read(nil, h, t.cols...) + if err != nil { + return nil, err + } + + nh = data[0].(int64) + for _, col := range t.cols { + m[col.name] = data[2+col.index] + } + m["$id"] = data[1] + if expr != nil { + val, err := s.where.eval(ctx, m, ctx.arg) + if err != nil { + return nil, err + } + + if val == nil { + continue + } + + x, ok := val.(bool) + if !ok { + return nil, fmt.Errorf("invalid WHERE expression %s (value of type %T)", val, val) + } + + if !x { + continue + } + } + + // hit + for i, asgn := range s.list { + val, err := asgn.expr.eval(ctx, m, ctx.arg) + if err != nil { + return nil, err + } + + colIndex := tcols[i].index + if t.hasIndices() { + old[colIndex] = data[2+colIndex] + touched[colIndex] = true + } + data[2+colIndex] = val + } + if err = typeCheck(data[2:], t.cols); err != nil { + return nil, err + } + + for i, v := range t.indices { + if i == 0 { // id() N/A + continue + } + + if v == nil || !touched[i-1] { + continue + } + + if err = v.x.Delete(old[i-1], h); err != nil { + return nil, err + } + } + + if err = t.store.UpdateRow(h, blobCols, data...); err != nil { //LATER detect which blobs are actually affected + return nil, err + } + + for i, v := range t.indices { + if i == 0 { // id() N/A + continue + } + + if v == nil || !touched[i-1] { + continue + } + + if err = v.x.Create(data[2+i-1], h); err != nil { + return nil, err + } + } + cc.RowsAffected++ + } + return +} + +func (s *updateStmt) isUpdating() bool { return true } + +type deleteStmt struct { + tableName string + where expression +} + +func (s *deleteStmt) String() string { + switch { + case s.where == nil: + return fmt.Sprintf("DELETE FROM %s;", s.tableName) + default: + return fmt.Sprintf("DELETE FROM %s WHERE %s;", s.tableName, s.where) + } +} + +func (s *deleteStmt) exec(ctx *execCtx) (_ Recordset, err error) { + t, ok := ctx.db.root.tables[s.tableName] + if !ok { + return nil, fmt.Errorf("DELETE FROM: table %s does not exist", s.tableName) + } + + m := map[interface{}]interface{}{} + var ph, h, nh int64 + var data []interface{} + blobCols := t.blobCols() + cc := ctx.db.cc + for h = t.head; h != 0; ph, h = h, nh { + for i, v := range data { + c, ok := v.(chunk) + if !ok { + continue + } + + data[i] = c.b + } + // Read can return lazily expanded chunks + data, err = t.store.Read(nil, h, t.cols...) + if err != nil { + return nil, err + } + + nh = data[0].(int64) + for _, col := range t.cols { + m[col.name] = data[2+col.index] + } + m["$id"] = data[1] + val, err := s.where.eval(ctx, m, ctx.arg) + if err != nil { + return nil, err + } + + if val == nil { + continue + } + + x, ok := val.(bool) + if !ok { + return nil, fmt.Errorf("invalid WHERE expression %s (value of type %T)", val, val) + } + + if !x { + continue + } + + // hit + for i, v := range t.indices { + if v == nil { + continue + } + + // overflow chunks left in place + if err = v.x.Delete(data[i+1], h); err != nil { + return nil, err + } + } + + // overflow chunks freed here + if err = t.store.Delete(h, blobCols...); err != nil { + return nil, err + } + + cc.RowsAffected++ + switch { + case ph == 0 && nh == 0: // "only" + fallthrough + case ph == 0 && nh != 0: // "first" + if err = t.store.Update(t.hhead, nh); err != nil { + return nil, err + } + + t.head, h = nh, 0 + case ph != 0 && nh == 0: // "last" + fallthrough + case ph != 0 && nh != 0: // "inner" + pdata, err := t.store.Read(nil, ph, t.cols...) + if err != nil { + return nil, err + } + + for i, v := range pdata { + if x, ok := v.(chunk); ok { + pdata[i] = x.b + } + } + pdata[0] = nh + if err = t.store.Update(ph, pdata...); err != nil { + return nil, err + } + + h = ph + } + } + + return +} + +func (s *deleteStmt) isUpdating() bool { return true } + +type truncateTableStmt struct { + tableName string +} + +func (s *truncateTableStmt) String() string { return fmt.Sprintf("TRUNCATE TABLE %s;", s.tableName) } + +func (s *truncateTableStmt) exec(ctx *execCtx) (Recordset, error) { + t, ok := ctx.db.root.tables[s.tableName] + if !ok { + return nil, fmt.Errorf("TRUNCATE TABLE: table %s does not exist", s.tableName) + } + + return nil, t.truncate() +} + +func (s *truncateTableStmt) isUpdating() bool { return true } + +type dropIndexStmt struct { + ifExists bool + indexName string +} + +func (s *dropIndexStmt) String() string { return fmt.Sprintf("DROP INDEX %s;", s.indexName) } + +func (s *dropIndexStmt) exec(ctx *execCtx) (Recordset, error) { + t, x := ctx.db.root.findIndexByName(s.indexName) + if x == nil { + if s.ifExists { + return nil, nil + } + + return nil, fmt.Errorf("DROP INDEX: index %s does not exist", s.indexName) + } + + for i, v := range t.indices { + if v == nil { + continue + } + + return nil, t.dropIndex(i) + } + + panic("internal error 058") +} + +func (s *dropIndexStmt) isUpdating() bool { return true } + +type dropTableStmt struct { + ifExists bool + tableName string +} + +func (s *dropTableStmt) String() string { return fmt.Sprintf("DROP TABLE %s;", s.tableName) } + +func (s *dropTableStmt) exec(ctx *execCtx) (Recordset, error) { + t, ok := ctx.db.root.tables[s.tableName] + if !ok { + if s.ifExists { + return nil, nil + } + + return nil, fmt.Errorf("DROP TABLE: table %s does not exist", s.tableName) + } + + return nil, ctx.db.root.dropTable(t) +} + +func (s *dropTableStmt) isUpdating() bool { return true } + +type alterTableDropColumnStmt struct { + tableName, colName string +} + +func (s *alterTableDropColumnStmt) String() string { + return fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s;", s.tableName, s.colName) +} + +func (s *alterTableDropColumnStmt) exec(ctx *execCtx) (Recordset, error) { + t, ok := ctx.db.root.tables[s.tableName] + if !ok { + return nil, fmt.Errorf("ALTER TABLE: table %s does not exist", s.tableName) + } + + cols := t.cols + for _, c := range cols { + if c.name == s.colName { + if len(cols) == 1 { + return nil, fmt.Errorf("ALTER TABLE %s DROP COLUMN: cannot drop the only column: %s", s.tableName, s.colName) + } + + c.name = "" + t.cols0[c.index].name = "" + if t.hasIndices() { + if v := t.indices[c.index+1]; v != nil { + if err := t.dropIndex(c.index + 1); err != nil { + return nil, err + } + } + } + return nil, t.updated() + } + } + + return nil, fmt.Errorf("ALTER TABLE %s DROP COLUMN: column %s does not exist", s.tableName, s.colName) +} + +func (s *alterTableDropColumnStmt) isUpdating() bool { return true } + +type alterTableAddStmt struct { + tableName string + c *col +} + +func (s *alterTableAddStmt) String() string { + return fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s;", s.tableName, s.c.name) +} + +func (s *alterTableAddStmt) exec(ctx *execCtx) (Recordset, error) { + t, ok := ctx.db.root.tables[s.tableName] + if !ok { + return nil, fmt.Errorf("ALTER TABLE: table %s does not exist", s.tableName) + } + + cols := t.cols + for _, c := range cols { + nm := c.name + if nm == s.c.name { + return nil, fmt.Errorf("ALTER TABLE %s ADD COLUMN %s: column exists", s.tableName, nm) + } + } + + if t.hasIndices() { + t.indices = append(t.indices, nil) + t.xroots = append(t.xroots, 0) + if err := t.store.Update(t.hxroots, t.xroots...); err != nil { + return nil, err + } + } + + t.cols0 = append(t.cols0, s.c) + return nil, t.updated() +} + +func (s *alterTableAddStmt) isUpdating() bool { return true } + +type selectStmt struct { + distinct bool + flds []*fld + from *crossJoinRset + group *groupByRset + hasAggregates bool + limit *limitRset + mu sync.Mutex + offset *offsetRset + order *orderByRset + where *whereRset +} + +func (s *selectStmt) String() string { + var b bytes.Buffer + b.WriteString("SELECT") + if s.distinct { + b.WriteString(" DISTINCT") + } + switch { + case len(s.flds) == 0: + b.WriteString(" *") + default: + a := make([]string, len(s.flds)) + for i, v := range s.flds { + s := v.expr.String() + if v.name != "" { + s += " AS " + v.name + } + a[i] = s + } + b.WriteString(" " + strings.Join(a, ", ")) + } + b.WriteString(" FROM ") + b.WriteString(s.from.String()) + if s.where != nil { + b.WriteString(" WHERE ") + b.WriteString(s.where.expr.String()) + } + if s.group != nil { + b.WriteString(" GROUP BY ") + b.WriteString(strings.Join(s.group.colNames, ", ")) + } + if s.order != nil { + b.WriteString(" ORDER BY ") + b.WriteString(s.order.String()) + } + if s.limit != nil { + b.WriteString(" LIMIT ") + b.WriteString(s.limit.expr.String()) + } + if s.offset != nil { + b.WriteString(" OFFSET ") + b.WriteString(s.offset.expr.String()) + } + b.WriteRune(';') + return b.String() +} + +func (s *selectStmt) do(ctx *execCtx, onlyNames bool, f func(id interface{}, data []interface{}) (more bool, err error)) (err error) { + return s.exec0().do(ctx, onlyNames, f) +} + +func (s *selectStmt) exec0() (r rset) { //LATER overlapping goroutines/pipelines + s.mu.Lock() + defer s.mu.Unlock() + r = rset(s.from) + if w := s.where; w != nil { + switch ok, list := isPossiblyRewriteableCrossJoinWhereExpression(w.expr); ok && len(s.from.sources) > 1 { + case true: + //dbg("====(in, %d)\n%s\n----", len(list), s) + tables := s.from.tables() + if len(list) != len(tables) { + r = &whereRset{expr: w.expr, src: r} + break + } + + m := map[string]int{} + for i, v := range tables { + m[v.name] = i + } + list2 := make([]int, len(list)) + for i, v := range list { + itab, ok := m[v.table] + if !ok { + break + } + + delete(m, v.table) + list2[i] = itab + if i == len(list)-1 { // last cycle + if len(m) != 0 { // all tabs "consumed" exactly once + break + } + + // Can rewrite + crs := s.from + for i, v := range list { + sel := &selectStmt{ + flds: []*fld{}, // SELECT * + from: &crossJoinRset{sources: []interface{}{[]interface{}{v.table, ""}}}, + where: &whereRset{expr: v.expr}, + } + info := tables[list2[i]] + crs.sources[info.i] = []interface{}{sel, info.rename} + } + r = rset(crs) + s.where = nil + //dbg("====(out)\n%s\n----", s) + } + } + if s.where == nil { + break + } + + fallthrough + default: + r = &whereRset{expr: w.expr, src: r} + } + } + switch { + case !s.hasAggregates && s.group == nil: // nop + case !s.hasAggregates && s.group != nil: + r = &groupByRset{colNames: s.group.colNames, src: r} + case s.hasAggregates && s.group == nil: + r = &groupByRset{src: r} + case s.hasAggregates && s.group != nil: + r = &groupByRset{colNames: s.group.colNames, src: r} + } + r = &selectRset{flds: s.flds, src: r} + if s.distinct { + r = &distinctRset{src: r} + } + if s := s.order; s != nil { + r = &orderByRset{asc: s.asc, by: s.by, src: r} + } + if s := s.offset; s != nil { + r = &offsetRset{s.expr, r} + } + if s := s.limit; s != nil { + r = &limitRset{s.expr, r} + } + return +} + +func (s *selectStmt) exec(ctx *execCtx) (rs Recordset, err error) { + return recordset{ctx, s.exec0(), nil}, nil +} + +func (s *selectStmt) isUpdating() bool { return false } + +type insertIntoStmt struct { + colNames []string + lists [][]expression + sel *selectStmt + tableName string +} + +func (s *insertIntoStmt) String() string { + cn := "" + if len(s.colNames) != 0 { + cn = fmt.Sprintf(" (%s)", strings.Join(s.colNames, ", ")) + } + switch { + case s.sel != nil: + return fmt.Sprintf("INSERT INTO %s%s (%s);", s.tableName, cn, s.sel) + default: + a := make([]string, len(s.lists)) + for i, v := range s.lists { + b := make([]string, len(v)) + for i, v := range v { + b[i] = v.String() + } + a[i] = fmt.Sprintf("(%s)", strings.Join(b, ", ")) + } + return fmt.Sprintf("INSERT INTO %s%s VALUES %s;", s.tableName, cn, strings.Join(a, ", ")) + } +} + +func (s *insertIntoStmt) execSelect(t *table, cols []*col, ctx *execCtx) (_ Recordset, err error) { + r := s.sel.exec0() + ok := false + h := t.head + data0 := make([]interface{}, len(t.cols0)+2) + cc := ctx.db.cc + if err = r.do(ctx, false, func(id interface{}, data []interface{}) (more bool, err error) { + if ok { + for i, d := range data { + data0[cols[i].index+2] = d + } + if err = typeCheck(data0[2:], cols); err != nil { + return + } + + id, err := t.store.ID() + if err != nil { + return false, err + } + + data0[0] = h + data0[1] = id + + // Any overflow chunks are written here. + if h, err = t.store.Create(data0...); err != nil { + return false, err + } + + for i, v := range t.indices { + if v == nil { + continue + } + + // Any overflow chunks are shared with the BTree key + if err = v.x.Create(data0[i+1], h); err != nil { + return false, err + } + } + + cc.RowsAffected++ + ctx.db.root.lastInsertID = id + return true, nil + } + + ok = true + flds := data[0].([]*fld) + if g, e := len(flds), len(cols); g != e { + return false, fmt.Errorf("INSERT INTO SELECT: mismatched column counts, have %d, need %d", g, e) + } + + return true, nil + }); err != nil { + return + } + + if err = t.store.Update(t.hhead, h); err != nil { + return + } + + t.head = h + return +} + +func (s *insertIntoStmt) exec(ctx *execCtx) (_ Recordset, err error) { + t, ok := ctx.db.root.tables[s.tableName] + if !ok { + return nil, fmt.Errorf("INSERT INTO %s: table does not exist", s.tableName) + } + + var cols []*col + switch len(s.colNames) { + case 0: + cols = t.cols + default: + for _, colName := range s.colNames { + if col := findCol(t.cols, colName); col != nil { + cols = append(cols, col) + continue + } + + return nil, fmt.Errorf("INSERT INTO %s: unknown column %s", s.tableName, colName) + } + } + + if s.sel != nil { + return s.execSelect(t, cols, ctx) + } + + for _, list := range s.lists { + if g, e := len(list), len(cols); g != e { + return nil, fmt.Errorf("INSERT INTO %s: expected %d value(s), have %d", s.tableName, e, g) + } + } + + arg := ctx.arg + root := ctx.db.root + cc := ctx.db.cc + r := make([]interface{}, len(t.cols0)) + for _, list := range s.lists { + for i, expr := range list { + val, err := expr.eval(ctx, nil, arg) + if err != nil { + return nil, err + } + + r[cols[i].index] = val + } + if err = typeCheck(r, cols); err != nil { + return + } + + id, err := t.addRecord(r) + if err != nil { + return nil, err + } + + cc.RowsAffected++ + root.lastInsertID = id + } + return +} + +func (s *insertIntoStmt) isUpdating() bool { return true } + +type beginTransactionStmt struct{} + +func (beginTransactionStmt) String() string { return "BEGIN TRANSACTION;" } +func (beginTransactionStmt) exec(*execCtx) (Recordset, error) { + log.Panic("internal error 059") + panic("unreachable") +} +func (beginTransactionStmt) isUpdating() bool { + log.Panic("internal error 060") + panic("unreachable") +} + +type commitStmt struct{} + +func (commitStmt) String() string { return "COMMIT;" } +func (commitStmt) exec(*execCtx) (Recordset, error) { + log.Panic("internal error 061") + panic("unreachable") +} +func (commitStmt) isUpdating() bool { + log.Panic("internal error 062") + panic("unreachable") +} + +type rollbackStmt struct{} + +func (rollbackStmt) String() string { return "ROLLBACK;" } +func (rollbackStmt) exec(*execCtx) (Recordset, error) { + log.Panic("internal error 063") + panic("unreachable") +} +func (rollbackStmt) isUpdating() bool { + log.Panic("internal error 064") + panic("unreachable") +} + +type createIndexStmt struct { + colName string // alt. "id()" for index on id() + ifNotExists bool + indexName string + tableName string + unique bool +} + +func (s *createIndexStmt) String() string { + u := "" + if s.unique { + u = "UNIQUE " + } + e := "" + if s.ifNotExists { + e = "IF NOT EXISTS " + } + return fmt.Sprintf("CREATE %sINDEX %s%s ON %s (%s);", u, e, s.indexName, s.tableName, s.colName) +} + +func (s *createIndexStmt) exec(ctx *execCtx) (Recordset, error) { + root := ctx.db.root + if t, i := root.findIndexByName(s.indexName); i != nil { + if s.ifNotExists { + return nil, nil + } + + return nil, fmt.Errorf("CREATE INDEX: table %s already has an index named %s", t.name, i.name) + } + + if root.tables[s.indexName] != nil { + return nil, fmt.Errorf("CREATE INDEX: index name collision with existing table: %s", s.indexName) + } + + t, ok := root.tables[s.tableName] + if !ok { + return nil, fmt.Errorf("CREATE INDEX: table does not exist %s", s.tableName) + } + + if findCol(t.cols, s.indexName) != nil { + return nil, fmt.Errorf("CREATE INDEX: index name collision with existing column: %s", s.indexName) + } + + if s.colName == "id()" { + if err := t.addIndex(s.unique, s.indexName, -1); err != nil { + return nil, fmt.Errorf("CREATE INDEX: %v", err) + } + + return nil, t.updated() + } + + c := findCol(t.cols, s.colName) + if c == nil { + return nil, fmt.Errorf("CREATE INDEX: column does not exist: %s", s.colName) + } + + if err := t.addIndex(s.unique, s.indexName, c.index); err != nil { + return nil, fmt.Errorf("CREATE INDEX: %v", err) + } + + return nil, t.updated() +} + +func (s *createIndexStmt) isUpdating() bool { return true } + +type createTableStmt struct { + ifNotExists bool + tableName string + cols []*col +} + +func (s *createTableStmt) String() string { + a := make([]string, len(s.cols)) + for i, v := range s.cols { + a[i] = fmt.Sprintf("%s %s", v.name, typeStr(v.typ)) + } + e := "" + if s.ifNotExists { + e = "IF NOT EXISTS " + } + return fmt.Sprintf("CREATE TABLE %s%s (%s);", e, s.tableName, strings.Join(a, ", ")) +} + +func (s *createTableStmt) exec(ctx *execCtx) (_ Recordset, err error) { + root := ctx.db.root + if _, ok := root.tables[s.tableName]; ok { + if s.ifNotExists { + return nil, nil + } + + return nil, fmt.Errorf("CREATE TABLE: table exists %s", s.tableName) + } + + if t, x := root.findIndexByName(s.tableName); x != nil { + return nil, fmt.Errorf("CREATE TABLE: table %s has index %s", t.name, s.tableName) + } + + m := map[string]bool{} + for i, c := range s.cols { + nm := c.name + if m[nm] { + return nil, fmt.Errorf("CREATE TABLE: duplicate column %s", nm) + } + + m[nm] = true + c.index = i + } + _, err = root.createTable(s.tableName, s.cols) + return +} + +func (s *createTableStmt) isUpdating() bool { return true } diff --git a/Godeps/_workspace/src/github.com/cznic/ql/storage.go b/Godeps/_workspace/src/github.com/cznic/ql/storage.go new file mode 100644 index 00000000000..7b6dc4723c6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/storage.go @@ -0,0 +1,669 @@ +// Copyright (c) 2014 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "fmt" + "log" + "strings" +) + +type storage interface { + Acid() bool + BeginTransaction() error + Close() error + Commit() error + Create(data ...interface{}) (h int64, err error) + CreateIndex(unique bool) (handle int64, x btreeIndex, err error) + CreateTemp(asc bool) (bt temp, err error) + Delete(h int64, blobCols ...*col) error //LATER split the nil blobCols case + ID() (id int64, err error) + Name() string + OpenIndex(unique bool, handle int64) (btreeIndex, error) // Never called on the memory backend. + Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) + ResetID() (err error) + Rollback() error + Update(h int64, data ...interface{}) error + UpdateRow(h int64, blobCols []*col, data ...interface{}) error + Verify() (allocs int64, err error) +} + +type btreeIterator interface { + Next() (k, v []interface{}, err error) +} + +type temp interface { + BeginTransaction() error + Create(data ...interface{}) (h int64, err error) + Drop() (err error) + Get(k []interface{}) (v []interface{}, err error) + Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) + SeekFirst() (e btreeIterator, err error) + Set(k, v []interface{}) (err error) +} + +type indexIterator interface { + Next() (k interface{}, h int64, err error) + Prev() (k interface{}, h int64, err error) +} + +type btreeIndex interface { + Clear() error // supports truncate table statement + Create(indexedValue interface{}, h int64) error // supports insert into statement + Delete(indexedValue interface{}, h int64) error // supports delete from statement + Drop() error // supports drop table, drop index statements + Seek(indexedValue interface{}) (iter indexIterator, hit bool, err error) // supports where clause + SeekFirst() (iter indexIterator, err error) // supports aggregate min / ascending order by + SeekLast() (iter indexIterator, err error) // supports aggregate max / descending order by +} + +type indexedCol struct { + name string + unique bool + x btreeIndex + xroot int64 +} + +type indexKey struct { + value interface{} + h int64 +} + +// storage fields +// 0: next int64 +// 1: scols string +// 2: hhead int64 +// 3: name string +// 4: indices string - optional +// 5: hxroots int64 - optional +type table struct { + cols []*col // logical + cols0 []*col // physical + h int64 // + head int64 // head of the single linked record list + hhead int64 // handle of the head of the single linked record list + hxroots int64 + indices []*indexedCol + name string + next int64 // single linked table list + store storage + tnext *table + tprev *table + xroots []interface{} +} + +func (t *table) hasIndices() bool { return len(t.indices) != 0 } + +func (t *table) clone() *table { + r := &table{} + *r = *t + r.cols = make([]*col, len(t.cols)) + for i, v := range t.cols { + c := &col{} + *c = *v + r.cols[i] = c + } + r.cols0 = make([]*col, len(t.cols0)) + for i, v := range t.cols0 { + c := &col{} + *c = *v + r.cols0[i] = c + } + r.indices = make([]*indexedCol, len(t.indices)) + for i, v := range t.indices { + if v != nil { + c := &indexedCol{} + *c = *v + r.indices[i] = c + } + } + r.xroots = make([]interface{}, len(t.xroots)) + copy(r.xroots, t.xroots) + r.tnext, r.tprev = nil, nil + return r +} + +func (t *table) findIndexByName(name string) *indexedCol { + for _, v := range t.indices { + if v != nil && v.name == name { + return v + } + } + return nil +} + +func (t *table) load() (err error) { + data, err := t.store.Read(nil, t.h) + if err != nil { + return + } + + var hasIndices bool + switch n := len(data); n { + case 4: + case 6: + hasIndices = true + default: + return fmt.Errorf("corrupted DB: table data len %d", n) + } + + var ok bool + if t.next, ok = data[0].(int64); !ok { + return fmt.Errorf("corrupted DB: table data[0] of type %T", data[0]) + } + + scols, ok := data[1].(string) + if !ok { + return fmt.Errorf("corrupted DB: table data[1] of type %T", data[1]) + } + + if t.hhead, ok = data[2].(int64); !ok { + return fmt.Errorf("corrupted DB: table data[2] of type %T", data[2]) + } + + if t.name, ok = data[3].(string); !ok { + return fmt.Errorf("corrupted DB: table data[3] of type %T", data[3]) + } + + var head []interface{} + if head, err = t.store.Read(nil, t.hhead); err != nil { + return err + } + + if len(head) != 1 { + return fmt.Errorf("corrupted DB: table head data len %d", len(head)) + } + + if t.head, ok = head[0].(int64); !ok { + return fmt.Errorf("corrupted DB: table head data[0] of type %T", head[0]) + } + + a := strings.Split(scols, "|") + t.cols0 = make([]*col, len(a)) + for i, v := range a { + if len(v) < 1 { + return fmt.Errorf("corrupted DB: field info %q", v) + } + + col := &col{name: v[1:], typ: int(v[0]), index: i} + t.cols0[i] = col + if col.name != "" { + t.cols = append(t.cols, col) + } + } + + if !hasIndices { + return + } + + if t.hxroots, ok = data[5].(int64); !ok { + return fmt.Errorf("corrupted DB: table data[5] of type %T", data[5]) + } + + xroots, err := t.store.Read(nil, t.hxroots) + if err != nil { + return err + } + + if g, e := len(xroots), len(t.cols0)+1; g != e { + return fmt.Errorf("corrupted DB: got %d index roots, expected %d", g, e) + } + + indices, ok := data[4].(string) + if !ok { + return fmt.Errorf("corrupted DB: table data[4] of type %T", data[4]) + } + + a = strings.Split(indices, "|") + if g, e := len(a), len(t.cols0)+1; g != e { + return fmt.Errorf("corrupted DB: got %d index definitions, expected %d", g, e) + } + + t.indices = make([]*indexedCol, len(a)) + for i, v := range a { + if v == "" { + continue + } + + if len(v) < 2 { + return fmt.Errorf("corrupted DB: invalid index definition %q", v) + } + + nm := v[1:] + h, ok := xroots[i].(int64) + if !ok { + return fmt.Errorf("corrupted DB: table index root of type %T", xroots[i]) + } + + if h == 0 { + return fmt.Errorf("corrupted DB: missing root for index %s", nm) + } + + unique := v[0] == 'u' + x, err := t.store.OpenIndex(unique, h) + if err != nil { + return err + } + + t.indices[i] = &indexedCol{nm, unique, x, h} + } + t.xroots = xroots + + return +} + +func newTable(store storage, name string, next int64, cols []*col, tprev, tnext *table) (t *table, err error) { + hhead, err := store.Create(int64(0)) + if err != nil { + return + } + + scols := cols2meta(cols) + h, err := store.Create(next, scols, hhead, name) + if err != nil { + return + } + + t = &table{ + cols0: cols, + h: h, + hhead: hhead, + name: name, + next: next, + store: store, + tnext: tnext, + tprev: tprev, + } + return t.updateCols(), nil +} + +func (t *table) blobCols() (r []*col) { + for _, c := range t.cols0 { + switch c.typ { + case qBlob, qBigInt, qBigRat, qTime, qDuration: + r = append(r, c) + } + } + return +} + +func (t *table) truncate() (err error) { + h := t.head + var rec []interface{} + blobCols := t.blobCols() + for h != 0 { + rec, err := t.store.Read(rec, h) + if err != nil { + return err + } + nh := rec[0].(int64) + + if err = t.store.Delete(h, blobCols...); err != nil { //LATER remove double read for len(blobCols) != 0 + return err + } + + h = nh + } + if err = t.store.Update(t.hhead, 0); err != nil { + return + } + + for _, v := range t.indices { + if v == nil { + continue + } + + if err := v.x.Clear(); err != nil { + return err + } + } + t.head = 0 + return t.updated() +} + +func (t *table) addIndex0(unique bool, indexName string, colIndex int) (btreeIndex, error) { + switch len(t.indices) { + case 0: + indices := make([]*indexedCol, len(t.cols0)+1) + h, x, err := t.store.CreateIndex(unique) + if err != nil { + return nil, err + } + + indices[colIndex+1] = &indexedCol{indexName, unique, x, h} + xroots := make([]interface{}, len(indices)) + xroots[colIndex+1] = h + hx, err := t.store.Create(xroots...) + if err != nil { + return nil, err + } + + t.hxroots, t.xroots, t.indices = hx, xroots, indices + return x, t.updated() + default: + ex := t.indices[colIndex+1] + if ex != nil && ex.name != "" { + colName := "id()" + if colIndex >= 0 { + colName = t.cols0[colIndex].name + } + return nil, fmt.Errorf("column %s already has an index: %s", colName, ex.name) + } + + h, x, err := t.store.CreateIndex(unique) + if err != nil { + return nil, err + } + + t.xroots[colIndex+1] = h + if err := t.store.Update(t.hxroots, t.xroots...); err != nil { + return nil, err + } + + t.indices[colIndex+1] = &indexedCol{indexName, unique, x, h} + return x, t.updated() + } +} + +func (t *table) addIndex(unique bool, indexName string, colIndex int) error { + x, err := t.addIndex0(unique, indexName, colIndex) + if err != nil { + return err + } + + // Must fill the new index. + ncols := len(t.cols0) + h, store := t.head, t.store + for h != 0 { + rec, err := store.Read(nil, h, t.cols...) + if err != nil { + return err + } + + if n := ncols + 2 - len(rec); n > 0 { + rec = append(rec, make([]interface{}, n)...) + } + + if err = x.Create(rec[colIndex+2], h); err != nil { + return err + } + + h = rec[0].(int64) + } + return nil +} + +func (t *table) dropIndex(xIndex int) error { + t.xroots[xIndex] = 0 + if err := t.indices[xIndex].x.Drop(); err != nil { + return err + } + + t.indices[xIndex] = nil + return t.updated() +} + +func (t *table) updated() (err error) { + switch { + case len(t.indices) != 0: + a := []string{} + for _, v := range t.indices { + if v == nil { + a = append(a, "") + continue + } + + s := "n" + if v.unique { + s = "u" + } + a = append(a, s+v.name) + } + return t.store.Update(t.h, t.next, cols2meta(t.updateCols().cols0), t.hhead, t.name, strings.Join(a, "|"), t.hxroots) + default: + return t.store.Update(t.h, t.next, cols2meta(t.updateCols().cols0), t.hhead, t.name) + } +} + +// storage fields +// 0: next record handle int64 +// 1: record id int64 +// 2...: data row +func (t *table) addRecord(r []interface{}) (id int64, err error) { + if id, err = t.store.ID(); err != nil { + return + } + + r = append([]interface{}{t.head, id}, r...) + h, err := t.store.Create(r...) + if err != nil { + return + } + + for i, v := range t.indices { + if v == nil { + continue + } + + if err = v.x.Create(r[i+1], h); err != nil { + return + } + } + + if err = t.store.Update(t.hhead, h); err != nil { + return + } + + t.head = h + return +} + +func (t *table) flds() (r []*fld) { + r = make([]*fld, len(t.cols)) + for i, v := range t.cols { + r[i] = &fld{expr: &ident{v.name}, name: v.name} + } + return +} + +func (t *table) updateCols() *table { + t.cols = t.cols[:0] + for i, c := range t.cols0 { + if c.name != "" { + c.index = i + t.cols = append(t.cols, c) + } + } + return t +} + +// storage fields +// 0: handle of first table in DB int64 +type root struct { + head int64 // Single linked table list + lastInsertID int64 + parent *root + rowsAffected int64 //LATER implement + store storage + tables map[string]*table + thead *table +} + +func newRoot(store storage) (r *root, err error) { + data, err := store.Read(nil, 1) + if err != nil { + return + } + + switch len(data) { + case 0: // new empty DB, create empty table list + if err = store.BeginTransaction(); err != nil { + return + } + + if err = store.Update(1, int64(0)); err != nil { + store.Rollback() + return + } + + if err = store.Commit(); err != nil { + return + } + + return &root{ + store: store, + tables: map[string]*table{}, + }, nil + case 1: // existing DB, load tables + if len(data) != 1 { + return nil, fmt.Errorf("corrupted DB") //LATER these messages must be distinct + } + + p, ok := data[0].(int64) + if !ok { + return nil, fmt.Errorf("corrupted DB") + } + + r := &root{ + head: p, + store: store, + tables: map[string]*table{}, + } + + var tprev *table + for p != 0 { + t := &table{ + h: p, + store: store, + tprev: tprev, + } + + if r.thead == nil { + r.thead = t + } + if tprev != nil { + tprev.tnext = t + } + tprev = t + + if err = t.load(); err != nil { + return nil, err + } + + if r.tables[t.name] != nil { // duplicate + return nil, fmt.Errorf("corrupted DB") + } + + r.tables[t.name] = t + p = t.next + } + return r, nil + default: + return nil, errIncompatibleDBFormat + } +} + +func (r *root) findIndexByName(name string) (*table, *indexedCol) { + for _, t := range r.tables { + if i := t.findIndexByName(name); i != nil { + return t, i + } + } + + return nil, nil +} + +func (r *root) updated() (err error) { + return r.store.Update(1, r.head) +} + +func (r *root) createTable(name string, cols []*col) (t *table, err error) { + if _, ok := r.tables[name]; ok { + log.Panic("internal error 065") + } + + if t, err = newTable(r.store, name, r.head, cols, nil, r.thead); err != nil { + return nil, err + } + + if err = r.store.Update(1, t.h); err != nil { + return nil, err + } + + if p := r.thead; p != nil { + p.tprev = t + } + r.tables[name], r.head, r.thead = t, t.h, t + return +} + +func (r *root) dropTable(t *table) (err error) { + defer func() { + if err != nil { + return + } + + delete(r.tables, t.name) + }() + + if err = t.truncate(); err != nil { + return + } + + if err = t.store.Delete(t.hhead); err != nil { + return + } + + if err = t.store.Delete(t.h); err != nil { + return + } + + for _, v := range t.indices { + if v != nil && v.x != nil { + if err = v.x.Drop(); err != nil { + return + } + } + } + + if h := t.hxroots; h != 0 { + if err = t.store.Delete(h); err != nil { + return + } + } + + switch { + case t.tprev == nil && t.tnext == nil: + r.head = 0 + r.thead = nil + err = r.updated() + return errSet(&err, r.store.ResetID()) + case t.tprev == nil && t.tnext != nil: + next := t.tnext + next.tprev = nil + r.head = next.h + r.thead = next + if err = r.updated(); err != nil { + return + } + + return next.updated() + case t.tprev != nil && t.tnext == nil: // last in list + prev := t.tprev + prev.next = 0 + prev.tnext = nil + return prev.updated() + default: //case t.tprev != nil && t.tnext != nil: + prev, next := t.tprev, t.tnext + prev.next = next.h + prev.tnext = next + next.tprev = prev + if err = prev.updated(); err != nil { + return + } + + return next.updated() + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/storage_test.go b/Godeps/_workspace/src/github.com/cznic/ql/storage_test.go new file mode 100644 index 00000000000..cc7b08e1356 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/storage_test.go @@ -0,0 +1,330 @@ +// Copyright (c) 2014 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "log" + "regexp" + "runtime/debug" + "strings" + "testing" +) + +var ( + oN = flag.Int("N", 0, "") + oM = flag.Int("M", 0, "") +) + +var testdata []string + +func init() { + tests, err := ioutil.ReadFile("testdata.ql") + if err != nil { + log.Panic(err) + } + + a := bytes.Split(tests, []byte("\n-- ")) + pre := []byte("-- ") + pres := []byte("S ") + for _, v := range a[1:] { + switch { + case bytes.HasPrefix(v, pres): + v = append(pre, v...) + v = append([]byte(sample), v...) + default: + v = append(pre, v...) + } + testdata = append(testdata, string(v)) + } +} + +func typeof(v interface{}) (r int) { //NTYPE + switch v.(type) { + case bool: + return qBool + case complex64: + return qComplex64 + case complex128: + return qComplex128 + case float32: + return qFloat32 + case float64: + return qFloat64 + case int8: + return qInt8 + case int16: + return qInt16 + case int32: + return qInt32 + case int64: + return qInt64 + case string: + return qString + case uint8: + return qUint8 + case uint16: + return qUint16 + case uint32: + return qUint32 + case uint64: + return qUint64 + } + return +} + +func stypeof(nm string, val interface{}) string { + if t := typeof(val); t != 0 { + return fmt.Sprintf("%c%s", t, nm) + } + + switch val.(type) { + case idealComplex: + return fmt.Sprintf("c%s", nm) + case idealFloat: + return fmt.Sprintf("f%s", nm) + case idealInt: + return fmt.Sprintf("l%s", nm) + case idealRune: + return fmt.Sprintf("k%s", nm) + case idealUint: + return fmt.Sprintf("x%s", nm) + default: + return fmt.Sprintf("?%s", nm) + } +} + +func dumpCols(cols []*col) string { + a := []string{} + for _, col := range cols { + a = append(a, fmt.Sprintf("%d:%s %s", col.index, col.name, typeStr(col.typ))) + } + return strings.Join(a, ",") +} + +func dumpFlds(flds []*fld) string { + a := []string{} + for _, fld := range flds { + a = append(a, fmt.Sprintf("%s AS %s", fld.expr, fld.name)) + } + return strings.Join(a, ",") +} + +func recSetDump(rs Recordset) (s string, err error) { + var state int + var a []string + var flds []*fld + rs2 := rs.(recordset) + if err = rs2.do(rs2.ctx, false, func(_ interface{}, rec []interface{}) (bool, error) { + switch state { + case 0: + flds = rec[0].([]*fld) + state++ + case 1: + for i, v := range flds { + a = append(a, stypeof(v.name, rec[i])) + } + a = []string{strings.Join(a, ", ")} + state++ + fallthrough + default: + if err = expand(rec); err != nil { + return false, err + } + + a = append(a, fmt.Sprintf("%v", rec)) + } + return true, nil + }); err != nil { + return + } + + if state == 1 { + for _, v := range flds { + a = append(a, stypeof(v.name, nil)) + } + a = []string{strings.Join(a, ", ")} + } + return strings.Join(a, "\n"), nil +} + +// http://en.wikipedia.org/wiki/Join_(SQL)#Sample_tables +const sample = ` + BEGIN TRANSACTION; + CREATE TABLE department ( + DepartmentID int, + DepartmentName string, + ); + + INSERT INTO department VALUES + (31, "Sales"), + (33, "Engineering"), + (34, "Clerical"), + (35, "Marketing"), + ; + + CREATE TABLE employee ( + LastName string, + DepartmentID int, + ); + + INSERT INTO employee VALUES + ("Rafferty", 31), + ("Jones", 33), + ("Heisenberg", 33), + ("Robinson", 34), + ("Smith", 34), + ("John", NULL), + ; + COMMIT; +` + +// Test provides a testing facility for alternative storage implementations. +// The storef should return freshly created and empty storage. Removing the +// store from the system is the responsibility of the caller. The test only +// guarantees not to panic on recoverable errors and return an error instead. +// Test errors are not returned but reported to t. +func test(t *testing.T, s testDB) (panicked error) { + defer func() { + if e := recover(); e != nil { + switch x := e.(type) { + case error: + panicked = x + default: + panicked = fmt.Errorf("%v", e) + } + } + if panicked != nil { + t.Errorf("PANIC: %v\n%s", panicked, debug.Stack()) + } + }() + + db, err := s.setup() + if err != nil { + t.Error(err) + return + } + + if err = s.mark(); err != nil { + t.Error(err) + return + } + + defer func() { + if err = s.teardown(); err != nil { + t.Error(err) + } + }() + + chk := func(test int, err error, expErr string, re *regexp.Regexp) (ok bool) { + s := err.Error() + if re == nil { + t.Error("FAIL: ", test, s) + return false + } + + if !re.MatchString(s) { + t.Error("FAIL: ", test, "error doesn't match:", s, "expected", expErr) + return false + } + + return true + } + + max := len(testdata) + if n := *oM; n != 0 && n < max { + max = n + } + for itest, test := range testdata[*oN:max] { + //dbg("---------------------------------------- itest %d", itest) + var re *regexp.Regexp + a := strings.Split(test+"|", "|") + q, rset := a[0], strings.TrimSpace(a[1]) + var expErr string + if len(a) < 3 { + t.Error(itest, "internal error 066") + return + } + + if expErr = a[2]; expErr != "" { + re = regexp.MustCompile("(?i:" + strings.TrimSpace(expErr) + ")") + } + + q = strings.Replace(q, "∨", "|", -1) + q = strings.Replace(q, "⩖", "||", -1) + list, err := Compile(q) + if err != nil { + if !chk(itest, err, expErr, re) { + return + } + + continue + } + + tctx := NewRWCtx() + if !func() (ok bool) { + defer func() { + nfo, err := db.Info() + if err != nil { + panic(err) + } + + for _, tab := range nfo.Tables { + if _, _, err = db.Run(NewRWCtx(), fmt.Sprintf(` + BEGIN TRANSACTION; + DROP table %s; + COMMIT; + `, + tab.Name)); err != nil { + panic(err) + } + } + }() + + if err = s.mark(); err != nil { + t.Error(err) + return + } + + rs, _, err := db.Execute(tctx, list, int64(30)) + if err != nil { + return chk(itest, err, expErr, re) + } + + if rs == nil { + t.Errorf("FAIL: %d: expected non nil Recordset or error %q", itest, expErr) + return + } + + g, err := recSetDump(rs[len(rs)-1]) + if err != nil { + return chk(itest, err, expErr, re) + } + + if expErr != "" { + t.Errorf("FAIL: %d: expected error %q", itest, expErr) + return + } + + a = strings.Split(rset, "\n") + for i, v := range a { + a[i] = strings.TrimSpace(v) + } + e := strings.Join(a, "\n") + if g != e { + t.Errorf("FAIL: test # %d\n%s\n---- g\n%s\n---- e\n%s\n----", itest, q, g, e) + return + } + + return true + }() { + return + } + } + return +} diff --git a/Godeps/_workspace/src/github.com/cznic/ql/testdata.ql b/Godeps/_workspace/src/github.com/cznic/ql/testdata.ql new file mode 100644 index 00000000000..e186cd6e08f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/ql/testdata.ql @@ -0,0 +1,9462 @@ +// Copyright (c) 2014 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// vi:filetype=sql + +-- 0 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int); + INSERT INTO t VALUES(11, 22, 33); +COMMIT; +SELECT * FROM t; +|lc1, lc2, lc3 +[11 22 33] + +-- 1 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int); + CREATE TABLE t (c1 int); +COMMIT; +||table.*exists + +-- 2 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c1 int, c4 int); +COMMIT; +||duplicate column + +-- 3 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int); + ALTER TABLE t ADD c4 string; + INSERT INTO t VALUES (1, 2, 3, "foo"); +COMMIT; +SELECT * FROM t; +|lc1, lc2, lc3, sc4 +[1 2 3 foo] + +-- 4 +BEGIN TRANSACTION; + ALTER TABLE none ADD c1 int; +COMMIT; +||table .* not exist + +-- 5 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int); + ALTER TABLE t ADD c2 int; +COMMIT; +||column .* exists + +-- 6 +BEGIN TRANSACTION; + ALTER TABLE none DROP COLUMN c1; +COMMIT; +||table .* not exist + +-- 7 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int); + ALTER TABLE t DROP COLUMN c4; +COMMIT; +||column .* not exist + +-- 8 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int); + ALTER TABLE t DROP COLUMN c2; + INSERT INTO t VALUES (1, 2); +COMMIT; +SELECT * FROM t; +|lc1, lc3 +[1 2] + +-- 9 +BEGIN TRANSACTION; + DROP TABLE none; +COMMIT; +||table .* not exist + +-- 10 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int); + DROP TABLE t; +COMMIT; +SELECT * FROM t; +||table .* not exist + +-- 11 +BEGIN TRANSACTION; + INSERT INTO none VALUES (1, 2); +COMMIT; +||table .* not exist + +-- 12 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int); + INSERT INTO t VALUES (1); +COMMIT; +||expect + +-- 13 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int); + INSERT INTO t VALUES (1, 2, 3); +COMMIT; +||expect + +-- 14 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int); + INSERT INTO t VALUES (1, 2/(3*5-15)); +COMMIT; +||division by zero + +-- 15 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int); + INSERT INTO t VALUES (2+3*4, 2*3+4); +COMMIT; +SELECT * FROM t; +|lc1, lc2 +[14 10] + +-- 16 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int, c4 int); + INSERT INTO t (c2, c4) VALUES (1); +COMMIT; +||expect + +-- 17 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int, c4 int); + INSERT INTO t (c2, c4) VALUES (1, 2, 3); +COMMIT; +||expect + +-- 18 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int, c4 int); + INSERT INTO t (c2, none) VALUES (1, 2); +COMMIT; +||unknown + +-- 19 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int, c4 int); + INSERT INTO t (c2, c3) VALUES (2+3*4, 2*3+4); + INSERT INTO t VALUES (1, 2, 3, 4, ); +COMMIT; +SELECT * FROM t; +|lc1, lc2, lc3, lc4 +[1 2 3 4] +[ 14 10 ] + +-- 20 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 int, c3 int, c4 int); + ALTER TABLE t DROP COLUMN c3; + INSERT INTO t (c1, c4) VALUES (42, 314); + INSERT INTO t (c1, c2) VALUES (2+3*4, 2*3+4); + INSERT INTO t VALUES (1, 2, 3); +COMMIT; +SELECT * FROM t; +|lc1, lc2, lc4 +[1 2 3] +[14 10 ] +[42 314] + +-- 21 +BEGIN TRANSACTION; + TRUNCATE TABLE none; +COMMIT; +||table .* not exist + +-- 22 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int); + INSERT INTO t VALUES(278); + TRUNCATE TABLE t; + INSERT INTO t VALUES(314); +COMMIT; +SELECT * FROM t; +|lc1 +[314] + +-- 23 +SELECT * FROM none; +||table .* not exist + +-- 24 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (2, "b"); + INSERT INTO t VALUES (1, "a"); +COMMIT; +SELECT * FROM t; +|lc1, sc2 +[1 a] +[2 b] + +-- 25 +SELECT c1 FROM none; +||table .* not exist + +-- 26 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); +COMMIT; +SELECT none FROM t; +||unknown + +-- 27 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); +COMMIT; +SELECT c1, none, c2 FROM t; +||unknown + +-- 28 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int32, c2 string); + INSERT INTO t VALUES (2, "b"); + INSERT INTO t VALUES (1, "a"); +COMMIT; +SELECT 3*c1 AS v FROM t; +|kv +[3] +[6] + +-- 29 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (2, "b"); + INSERT INTO t VALUES (1, "a"); +COMMIT; +SELECT c2 FROM t; +|sc2 +[a] +[b] + +-- 30 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (2, "b"); + INSERT INTO t VALUES (1, "a"); +COMMIT; +SELECT c1 AS X, c2 FROM t; +|lX, sc2 +[1 a] +[2 b] + +-- 31 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (2, "b"); + INSERT INTO t VALUES (1, "a"); +COMMIT; +SELECT c2, c1 AS Y FROM t; +|sc2, lY +[a 1] +[b 2] + +-- 32 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); +COMMIT; +SELECT * FROM t WHERE c3 == 1; +||unknown + +-- 33 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); +COMMIT; +SELECT * FROM t WHERE c1 == 1; +|lc1, sc2 +[1 a] + +-- 34 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); +COMMIT; +SELECT * FROM t ORDER BY c3; +||unknown + +-- 35 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int32, c2 string); + INSERT INTO t VALUES (22, "bc"); + INSERT INTO t VALUES (11, "ab"); + INSERT INTO t VALUES (33, "cd"); +COMMIT; +SELECT * FROM t ORDER BY c1; +|kc1, sc2 +[11 ab] +[22 bc] +[33 cd] + +-- 36 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); +COMMIT; +SELECT * FROM t ORDER BY c1 ASC; +|lc1, sc2 +[1 a] +[2 b] + +-- 37 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); +COMMIT; +SELECT * FROM t ORDER BY c1 DESC; +|lc1, sc2 +[2 b] +[1 a] + +-- 38 +BEGIN TRANSACTION; +CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "b"); + INSERT INTO t VALUES (3, "c"); + INSERT INTO t VALUES (4, "d"); + INSERT INTO t VALUES (5, "e"); + INSERT INTO t VALUES (6, "f"); + INSERT INTO t VALUES (7, "g"); +COMMIT; +SELECT * FROM t +WHERE c1 % 2 == 0 +ORDER BY c2 DESC; +|lc1, sc2 +[6 f] +[4 d] +[2 b] + +-- 39 +BEGIN TRANSACTION; +CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (2, "a"); + INSERT INTO t VALUES (3, "b"); + INSERT INTO t VALUES (4, "b"); + INSERT INTO t VALUES (5, "c"); + INSERT INTO t VALUES (6, "c"); + INSERT INTO t VALUES (7, "d"); +COMMIT; +SELECT * FROM t +ORDER BY c1, c2; +|lc1, sc2 +[1 a] +[2 a] +[3 b] +[4 b] +[5 c] +[6 c] +[7 d] + +-- 40 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int, c2 string); + INSERT INTO t VALUES (1, "d"); + INSERT INTO t VALUES (2, "c"); + INSERT INTO t VALUES (3, "c"); + INSERT INTO t VALUES (4, "b"); + INSERT INTO t VALUES (5, "b"); + INSERT INTO t VALUES (6, "a"); + INSERT INTO t VALUES (7, "a"); +COMMIT; +SELECT * FROM t +ORDER BY c2, c1 +|lc1, sc2 +[6 a] +[7 a] +[4 b] +[5 b] +[2 c] +[3 c] +[1 d] + +-- S 41 +SELECT * FROM employee, none; +||table .* not exist + +-- S 42 +SELECT employee.LastName FROM employee, none; +||table .* not exist + +-- S 43 +SELECT none FROM employee, department; +||unknown + +-- S 44 +SELECT employee.LastName FROM employee, department; +|semployee.LastName +[John] +[John] +[John] +[John] +[Smith] +[Smith] +[Smith] +[Smith] +[Robinson] +[Robinson] +[Robinson] +[Robinson] +[Heisenberg] +[Heisenberg] +[Heisenberg] +[Heisenberg] +[Jones] +[Jones] +[Jones] +[Jones] +[Rafferty] +[Rafferty] +[Rafferty] +[Rafferty] + +-- S 45 +SELECT * FROM employee, department +ORDER by employee.LastName; +|semployee.LastName, lemployee.DepartmentID, ldepartment.DepartmentID, sdepartment.DepartmentName +[Heisenberg 33 35 Marketing] +[Heisenberg 33 34 Clerical] +[Heisenberg 33 33 Engineering] +[Heisenberg 33 31 Sales] +[John 35 Marketing] +[John 34 Clerical] +[John 33 Engineering] +[John 31 Sales] +[Jones 33 35 Marketing] +[Jones 33 34 Clerical] +[Jones 33 33 Engineering] +[Jones 33 31 Sales] +[Rafferty 31 35 Marketing] +[Rafferty 31 34 Clerical] +[Rafferty 31 33 Engineering] +[Rafferty 31 31 Sales] +[Robinson 34 35 Marketing] +[Robinson 34 34 Clerical] +[Robinson 34 33 Engineering] +[Robinson 34 31 Sales] +[Smith 34 35 Marketing] +[Smith 34 34 Clerical] +[Smith 34 33 Engineering] +[Smith 34 31 Sales] + +-- S 46 +SELECT * +FROM employee, department +WHERE employee.DepartmentID == department.DepartmentID; +|semployee.LastName, lemployee.DepartmentID, ldepartment.DepartmentID, sdepartment.DepartmentName +[Smith 34 34 Clerical] +[Robinson 34 34 Clerical] +[Heisenberg 33 33 Engineering] +[Jones 33 33 Engineering] +[Rafferty 31 31 Sales] + +-- S 47 +SELECT department.DepartmentName, department.DepartmentID, employee.LastName, employee.DepartmentID +FROM employee, department +WHERE employee.DepartmentID == department.DepartmentID +ORDER BY department.DepartmentName, employee.LastName; +|sdepartment.DepartmentName, ldepartment.DepartmentID, semployee.LastName, lemployee.DepartmentID +[Clerical 34 Robinson 34] +[Clerical 34 Smith 34] +[Engineering 33 Heisenberg 33] +[Engineering 33 Jones 33] +[Sales 31 Rafferty 31] + +-- S 48 +SELECT department.DepartmentName, department.DepartmentID, employee.LastName, employee.DepartmentID +FROM employee, department +WHERE department.DepartmentName IN ("Sales", "Engineering", "HR", "Clerical") +ORDER BY employee.LastName; +|sdepartment.DepartmentName, ldepartment.DepartmentID, semployee.LastName, lemployee.DepartmentID +[Clerical 34 Heisenberg 33] +[Engineering 33 Heisenberg 33] +[Sales 31 Heisenberg 33] +[Clerical 34 John ] +[Engineering 33 John ] +[Sales 31 John ] +[Clerical 34 Jones 33] +[Engineering 33 Jones 33] +[Sales 31 Jones 33] +[Clerical 34 Rafferty 31] +[Engineering 33 Rafferty 31] +[Sales 31 Rafferty 31] +[Clerical 34 Robinson 34] +[Engineering 33 Robinson 34] +[Sales 31 Robinson 34] +[Clerical 34 Smith 34] +[Engineering 33 Smith 34] +[Sales 31 Smith 34] + +-- S 49 +SELECT department.DepartmentName, department.DepartmentID, employee.LastName, employee.DepartmentID +FROM employee, department +WHERE (department.DepartmentID+1000) IN (1031, 1035, 1036) +ORDER BY employee.LastName; +|sdepartment.DepartmentName, ldepartment.DepartmentID, semployee.LastName, lemployee.DepartmentID +[Marketing 35 Heisenberg 33] +[Sales 31 Heisenberg 33] +[Marketing 35 John ] +[Sales 31 John ] +[Marketing 35 Jones 33] +[Sales 31 Jones 33] +[Marketing 35 Rafferty 31] +[Sales 31 Rafferty 31] +[Marketing 35 Robinson 34] +[Sales 31 Robinson 34] +[Marketing 35 Smith 34] +[Sales 31 Smith 34] + +-- S 50 +SELECT department.DepartmentName, department.DepartmentID, employee.LastName, employee.DepartmentID +FROM employee, department +WHERE department.DepartmentName NOT IN ("Engineering", "HR", "Clerical"); +|sdepartment.DepartmentName, ldepartment.DepartmentID, semployee.LastName, ?employee.DepartmentID +[Marketing 35 John ] +[Sales 31 John ] +[Marketing 35 Smith 34] +[Sales 31 Smith 34] +[Marketing 35 Robinson 34] +[Sales 31 Robinson 34] +[Marketing 35 Heisenberg 33] +[Sales 31 Heisenberg 33] +[Marketing 35 Jones 33] +[Sales 31 Jones 33] +[Marketing 35 Rafferty 31] +[Sales 31 Rafferty 31] + +-- S 51 +SELECT department.DepartmentName, department.DepartmentID, employee.LastName, employee.DepartmentID +FROM employee, department +WHERE department.DepartmentID BETWEEN 34 AND 36 +ORDER BY employee.LastName; +|sdepartment.DepartmentName, ldepartment.DepartmentID, semployee.LastName, lemployee.DepartmentID +[Marketing 35 Heisenberg 33] +[Clerical 34 Heisenberg 33] +[Marketing 35 John ] +[Clerical 34 John ] +[Marketing 35 Jones 33] +[Clerical 34 Jones 33] +[Marketing 35 Rafferty 31] +[Clerical 34 Rafferty 31] +[Marketing 35 Robinson 34] +[Clerical 34 Robinson 34] +[Marketing 35 Smith 34] +[Clerical 34 Smith 34] + +-- S 52 +SELECT department.DepartmentName, department.DepartmentID, employee.LastName, employee.DepartmentID +FROM employee, department +WHERE department.DepartmentID BETWEEN int64(34) AND int64(36) +ORDER BY employee.LastName; +|sdepartment.DepartmentName, ldepartment.DepartmentID, semployee.LastName, lemployee.DepartmentID +[Marketing 35 Heisenberg 33] +[Clerical 34 Heisenberg 33] +[Marketing 35 John ] +[Clerical 34 John ] +[Marketing 35 Jones 33] +[Clerical 34 Jones 33] +[Marketing 35 Rafferty 31] +[Clerical 34 Rafferty 31] +[Marketing 35 Robinson 34] +[Clerical 34 Robinson 34] +[Marketing 35 Smith 34] +[Clerical 34 Smith 34] + +-- S 53 +SELECT department.DepartmentName, department.DepartmentID, employee.LastName, employee.DepartmentID +FROM employee, department +WHERE department.DepartmentID NOT BETWEEN 33 AND 34 +ORDER BY employee.LastName; +|sdepartment.DepartmentName, ldepartment.DepartmentID, semployee.LastName, lemployee.DepartmentID +[Marketing 35 Heisenberg 33] +[Sales 31 Heisenberg 33] +[Marketing 35 John ] +[Sales 31 John ] +[Marketing 35 Jones 33] +[Sales 31 Jones 33] +[Marketing 35 Rafferty 31] +[Sales 31 Rafferty 31] +[Marketing 35 Robinson 34] +[Sales 31 Robinson 34] +[Marketing 35 Smith 34] +[Sales 31 Smith 34] + +-- S 54 +SELECT LastName, LastName FROM employee; +||duplicate + +-- S 55 +SELECT LastName+", " AS a, LastName AS a FROM employee; +||duplicate + +-- S 56 +SELECT LastName AS a, LastName AS b FROM employee +ORDER by a, b; +|sa, sb +[Heisenberg Heisenberg] +[John John] +[Jones Jones] +[Rafferty Rafferty] +[Robinson Robinson] +[Smith Smith] + +-- S 57 +SELECT employee.LastName AS name, employee.DepartmentID AS id, department.DepartmentName AS department, department.DepartmentID AS id2 +FROM employee, department +WHERE employee.DepartmentID == department.DepartmentID +ORDER BY name, id, department, id2; +|sname, lid, sdepartment, lid2 +[Heisenberg 33 Engineering 33] +[Jones 33 Engineering 33] +[Rafferty 31 Sales 31] +[Robinson 34 Clerical 34] +[Smith 34 Clerical 34] + +-- S 58 +SELECT * FROM; +||syntax + +-- S 59 +SELECT * FROM employee +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 33] +[John ] +[Jones 33] +[Rafferty 31] +[Robinson 34] +[Smith 34] + +-- S 60 +SELECT * FROM employee AS e +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 33] +[John ] +[Jones 33] +[Rafferty 31] +[Robinson 34] +[Smith 34] + +-- S 61 +SELECT none FROM ( + SELECT * FROM employee; + SELECT * FROM department; +); +||syntax + +-- S 62 +SELECT none FROM ( + SELECT * FROM employee; +); +||unknown + +-- S 63 +SELECT noneCol FROM ( + SELECT * FROM noneTab +); +||not exist + +-- S 64 +SELECT noneCol FROM ( + SELECT * FROM employee +); +||unknown + +-- S 65 +SELECT * FROM ( + SELECT * FROM employee +) +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 33] +[John ] +[Jones 33] +[Rafferty 31] +[Robinson 34] +[Smith 34] + +-- S 66 +SELECT * FROM ( + SELECT LastName AS Name FROM employee +) +ORDER BY Name; +|sName +[Heisenberg] +[John] +[Jones] +[Rafferty] +[Robinson] +[Smith] + +-- S 67 +SELECT Name FROM ( + SELECT LastName AS name FROM employee +); +||unknown + +-- S 68 +SELECT name AS Name FROM ( + SELECT LastName AS name + FROM employee AS e +) +ORDER BY Name; +|sName +[Heisenberg] +[John] +[Jones] +[Rafferty] +[Robinson] +[Smith] + +-- S 69 +SELECT name AS Name FROM ( + SELECT LastName AS name FROM employee +) +ORDER BY Name; +|sName +[Heisenberg] +[John] +[Jones] +[Rafferty] +[Robinson] +[Smith] + +-- S 70 +SELECT employee.LastName, department.DepartmentName, department.DepartmentID FROM ( + SELECT * + FROM employee, department + WHERE employee.DepartmentID == department.DepartmentID +) +ORDER BY department.DepartmentName, employee.LastName +|semployee.LastName, sdepartment.DepartmentName, ldepartment.DepartmentID +[Robinson Clerical 34] +[Smith Clerical 34] +[Heisenberg Engineering 33] +[Jones Engineering 33] +[Rafferty Sales 31] + +-- S 71 +SELECT e.LastName, d.DepartmentName, d.DepartmentID FROM ( + SELECT * + FROM employee AS e, department AS d + WHERE e.DepartmentID == d.DepartmentID +) +ORDER by d.DepartmentName, e.LastName; +|se.LastName, sd.DepartmentName, ld.DepartmentID +[Robinson Clerical 34] +[Smith Clerical 34] +[Heisenberg Engineering 33] +[Jones Engineering 33] +[Rafferty Sales 31] + +-- S 72 +SELECT e.LastName AS name, d.DepartmentName AS department, d.DepartmentID AS id FROM ( + SELECT * + FROM employee AS e, department AS d + WHERE e.DepartmentID == d.DepartmentID +) +ORDER by department, name +|sname, sdepartment, lid +[Robinson Clerical 34] +[Smith Clerical 34] +[Heisenberg Engineering 33] +[Jones Engineering 33] +[Rafferty Sales 31] + +-- S 73 +SELECT name, department, id FROM ( + SELECT e.LastName AS name, e.DepartmentID AS id, d.DepartmentName AS department, d.DepartmentID AS fid + FROM employee AS e, department AS d + WHERE e.DepartmentID == d.DepartmentID +) +ORDER by department, name; +|sname, sdepartment, lid +[Robinson Clerical 34] +[Smith Clerical 34] +[Heisenberg Engineering 33] +[Jones Engineering 33] +[Rafferty Sales 31] + +-- S 74 +SELECT * +FROM +( + SELECT * + FROM employee +), +( + SELECT * + FROM department +); +|s, ?, l, s +[John 35 Marketing] +[John 34 Clerical] +[John 33 Engineering] +[John 31 Sales] +[Smith 34 35 Marketing] +[Smith 34 34 Clerical] +[Smith 34 33 Engineering] +[Smith 34 31 Sales] +[Robinson 34 35 Marketing] +[Robinson 34 34 Clerical] +[Robinson 34 33 Engineering] +[Robinson 34 31 Sales] +[Heisenberg 33 35 Marketing] +[Heisenberg 33 34 Clerical] +[Heisenberg 33 33 Engineering] +[Heisenberg 33 31 Sales] +[Jones 33 35 Marketing] +[Jones 33 34 Clerical] +[Jones 33 33 Engineering] +[Jones 33 31 Sales] +[Rafferty 31 35 Marketing] +[Rafferty 31 34 Clerical] +[Rafferty 31 33 Engineering] +[Rafferty 31 31 Sales] + +-- S 75 +SELECT * +FROM +( + SELECT * + FROM employee +) AS e, +( + SELECT * + FROM department +) +ORDER BY e.LastName, e.DepartmentID; +|se.LastName, le.DepartmentID, l, s +[Heisenberg 33 35 Marketing] +[Heisenberg 33 34 Clerical] +[Heisenberg 33 33 Engineering] +[Heisenberg 33 31 Sales] +[John 35 Marketing] +[John 34 Clerical] +[John 33 Engineering] +[John 31 Sales] +[Jones 33 35 Marketing] +[Jones 33 34 Clerical] +[Jones 33 33 Engineering] +[Jones 33 31 Sales] +[Rafferty 31 35 Marketing] +[Rafferty 31 34 Clerical] +[Rafferty 31 33 Engineering] +[Rafferty 31 31 Sales] +[Robinson 34 35 Marketing] +[Robinson 34 34 Clerical] +[Robinson 34 33 Engineering] +[Robinson 34 31 Sales] +[Smith 34 35 Marketing] +[Smith 34 34 Clerical] +[Smith 34 33 Engineering] +[Smith 34 31 Sales] + +-- S 76 +SELECT * +FROM +( + SELECT * + FROM employee +), +( + SELECT * + FROM department +) AS d +ORDER BY d.DepartmentID DESC; +|s, l, ld.DepartmentID, sd.DepartmentName +[Rafferty 31 35 Marketing] +[Jones 33 35 Marketing] +[Heisenberg 33 35 Marketing] +[Robinson 34 35 Marketing] +[Smith 34 35 Marketing] +[John 35 Marketing] +[Rafferty 31 34 Clerical] +[Jones 33 34 Clerical] +[Heisenberg 33 34 Clerical] +[Robinson 34 34 Clerical] +[Smith 34 34 Clerical] +[John 34 Clerical] +[Rafferty 31 33 Engineering] +[Jones 33 33 Engineering] +[Heisenberg 33 33 Engineering] +[Robinson 34 33 Engineering] +[Smith 34 33 Engineering] +[John 33 Engineering] +[Rafferty 31 31 Sales] +[Jones 33 31 Sales] +[Heisenberg 33 31 Sales] +[Robinson 34 31 Sales] +[Smith 34 31 Sales] +[John 31 Sales] + +-- S 77 +SELECT * +FROM + employee, + ( + SELECT * + FROM department + ) +ORDER BY employee.LastName; +|semployee.LastName, lemployee.DepartmentID, l, s +[Heisenberg 33 35 Marketing] +[Heisenberg 33 34 Clerical] +[Heisenberg 33 33 Engineering] +[Heisenberg 33 31 Sales] +[John 35 Marketing] +[John 34 Clerical] +[John 33 Engineering] +[John 31 Sales] +[Jones 33 35 Marketing] +[Jones 33 34 Clerical] +[Jones 33 33 Engineering] +[Jones 33 31 Sales] +[Rafferty 31 35 Marketing] +[Rafferty 31 34 Clerical] +[Rafferty 31 33 Engineering] +[Rafferty 31 31 Sales] +[Robinson 34 35 Marketing] +[Robinson 34 34 Clerical] +[Robinson 34 33 Engineering] +[Robinson 34 31 Sales] +[Smith 34 35 Marketing] +[Smith 34 34 Clerical] +[Smith 34 33 Engineering] +[Smith 34 31 Sales] + +-- S 78 +SELECT * +FROM +( + SELECT * + FROM employee +) AS e, +( + SELECT * + FROM department +) AS d +WHERE e.DepartmentID == d.DepartmentID +ORDER BY d.DepartmentName, e.LastName; +|se.LastName, le.DepartmentID, ld.DepartmentID, sd.DepartmentName +[Robinson 34 34 Clerical] +[Smith 34 34 Clerical] +[Heisenberg 33 33 Engineering] +[Jones 33 33 Engineering] +[Rafferty 31 31 Sales] + +-- S 79 +SELECT * +FROM + employee, + ( + SELECT * + FROM department + ) AS d +WHERE employee.DepartmentID == d.DepartmentID +ORDER BY d.DepartmentName, employee.LastName; +|semployee.LastName, lemployee.DepartmentID, ld.DepartmentID, sd.DepartmentName +[Robinson 34 34 Clerical] +[Smith 34 34 Clerical] +[Heisenberg 33 33 Engineering] +[Jones 33 33 Engineering] +[Rafferty 31 31 Sales] + +-- S 80 +SELECT * +FROM + employee AS e, + ( + SELECT * + FROM department + ) AS d +WHERE e.DepartmentID == d.DepartmentID +ORDER BY d.DepartmentName, e.LastName; +|se.LastName, le.DepartmentID, ld.DepartmentID, sd.DepartmentName +[Robinson 34 34 Clerical] +[Smith 34 34 Clerical] +[Heisenberg 33 33 Engineering] +[Jones 33 33 Engineering] +[Rafferty 31 31 Sales] + +-- S 81 +SELECT * +FROM + employee AS e, + ( + SELECT * + FROM department + ) AS d +WHERE e.DepartmentID == d.DepartmentID == true +ORDER BY e.DepartmentID, e.LastName; +|se.LastName, le.DepartmentID, ld.DepartmentID, sd.DepartmentName +[Rafferty 31 31 Sales] +[Heisenberg 33 33 Engineering] +[Jones 33 33 Engineering] +[Robinson 34 34 Clerical] +[Smith 34 34 Clerical] + +-- S 82 +SELECT * +FROM + employee AS e, + ( + SELECT * + FROM department + ) AS d +WHERE e.DepartmentID != d.DepartmentID == false +ORDER BY e.DepartmentID, e.LastName; +|se.LastName, le.DepartmentID, ld.DepartmentID, sd.DepartmentName +[Rafferty 31 31 Sales] +[Heisenberg 33 33 Engineering] +[Jones 33 33 Engineering] +[Robinson 34 34 Clerical] +[Smith 34 34 Clerical] + +-- 83 +BEGIN TRANSACTION; + CREATE TABLE t (c1 bool); + INSERT INTO t VALUES (1); +COMMIT; +||cannot .* int64.*bool .* c1 + +-- 84 +BEGIN TRANSACTION; + CREATE TABLE t (c1 bool); + INSERT INTO t VALUES (true); +COMMIT; +SELECT * from t; +|bc1 +[true] + +-- 85 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int8); + INSERT INTO t VALUES ("foo"); +COMMIT; +||cannot .* string.*int8 .* c1 + +-- 86 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int8); + INSERT INTO t VALUES (0x1234); +COMMIT; +SELECT * from t; +||overflow + +-- 87 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int16); + INSERT INTO t VALUES (87); +COMMIT; +SELECT * from t; +|jc1 +[87] + +-- 88 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int16); + INSERT INTO t VALUES (int16(0x12345678)); +COMMIT; +SELECT * from t; +|jc1 +[22136] + +-- 89 +BEGIN TRANSACTION; +CREATE TABLE t (c1 int32); + INSERT INTO t VALUES (uint32(1)); +COMMIT; +||cannot .* uint32.*int32 .* c1 + +-- 90 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int32); + INSERT INTO t VALUES (0xabcd12345678); +COMMIT; +SELECT * from t; +||overflow + +-- 91 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int64); + INSERT INTO t VALUES (int8(1)); +COMMIT; +||cannot .* int8.*int64 .* c1 + +-- 92 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int64); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t; +|lc1 +[1] + +-- 93 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int); + INSERT INTO t VALUES (int8(1)); +COMMIT; +||cannot .* int8.*int64 .* c1 + +-- 94 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int); + INSERT INTO t VALUES (94); +COMMIT; +SELECT * from t; +|lc1 +[94] + +-- 95 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint8); + INSERT INTO t VALUES (95); +COMMIT; +SELECT * from t; +|uc1 +[95] + +-- 96 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint8); + INSERT INTO t VALUES (uint8(0x1234)); +COMMIT; +SELECT * from t; +|uc1 +[52] + +-- 97 +BEGIN TRANSACTION; + CREATE TABLE t (c1 byte); + INSERT INTO t VALUES (int8(1)); +COMMIT; +||cannot .* int8.*uint8 .* c1 + +-- 98 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint8); + INSERT INTO t VALUES (byte(0x1234)); +COMMIT; +SELECT * from t; +|uc1 +[52] + +-- 99 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint16); + INSERT INTO t VALUES (int(1)); +COMMIT; +||cannot .* int64.*uint16 .* c1 + +-- 100 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint16); + INSERT INTO t VALUES (0x12345678); +COMMIT; +SELECT * from t; +||overflow + +-- 101 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint32); + INSERT INTO t VALUES (int32(1)); +COMMIT; +||cannot .* int32.*uint32 .* c1 + +-- 102 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint32); + INSERT INTO t VALUES (uint32(0xabcd12345678)); +COMMIT; +SELECT * from t; +|wc1 +[305419896] + +-- 103 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint64); + INSERT INTO t VALUES (int(1)); +COMMIT; +||cannot .* int64.*uint64 .* c1 + +-- 104 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint64); + INSERT INTO t VALUES (uint64(1)); +COMMIT; +SELECT * from t; +|xc1 +[1] + +-- 105 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint); + INSERT INTO t VALUES (int(1)); +COMMIT; +||cannot .* int64.*uint64 .* c1 + +-- 106 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t; +|xc1 +[1] + +-- 107 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float32); + INSERT INTO t VALUES (107); +COMMIT; +SELECT * from t; +|fc1 +[107] + +-- 108 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float32); + INSERT INTO t VALUES (float64(1)); +COMMIT; +SELECT * from t; +||cannot .* float64.*float32 .* c1 + +-- 109 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float32); + INSERT INTO t VALUES (1.2); +COMMIT; +SELECT * from t; +|fc1 +[1.2] + +-- 110 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float64); + INSERT INTO t VALUES (1.2); +COMMIT; +SELECT * from t; +|gc1 +[1.2] + +-- 111 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float); + INSERT INTO t VALUES (111.1); +COMMIT; +SELECT * from t; +|gc1 +[111.1] + +-- 112 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float); + INSERT INTO t VALUES (-112.1); +COMMIT; +SELECT * from t; +|gc1 +[-112.1] + +-- 113 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex64); + INSERT INTO t VALUES (complex(1, 0.5)); +COMMIT; +SELECT * from t; +|cc1 +[(1+0.5i)] + +-- 114 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex64); + INSERT INTO t VALUES (complex128(complex(1, 0.5))); +COMMIT; +SELECT * from t; +||cannot .* complex128.*complex64 .* c1 + +-- 115 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex128); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t; +|dc1 +[(1+0i)] + +-- 116 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex128); + INSERT INTO t VALUES (complex(1, 0.5)); +COMMIT; +SELECT * from t; +|dc1 +[(1+0.5i)] + +-- 117 +BEGIN TRANSACTION; + CREATE TABLE t (c1 string); + INSERT INTO t VALUES (1); +COMMIT; +||cannot .* int64.*string .* c1 + +-- 118 +BEGIN TRANSACTION; + CREATE TABLE t (c1 string); + INSERT INTO t VALUES ("a"+"b"); +COMMIT; +SELECT * from t; +|sc1 +[ab] + +-- 119 +BEGIN TRANSACTION; + CREATE TABLE t (c1 bool); + INSERT INTO t VALUES (true); +COMMIT; +SELECT * from t +WHERE c1 > 3; +||operator .* not defined .* bool + +-- 120 +BEGIN TRANSACTION; + CREATE TABLE t (c1 bool); + INSERT INTO t VALUES (true); +COMMIT; +SELECT * from t +WHERE c1; +|bc1 +[true] + +-- 121 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int8); + INSERT INTO t VALUES (float(1)); +COMMIT; +SELECT * from t +WHERE c1 == 8; +||cannot .* float64.*int8 .* c1 + +-- 122 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int8); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == int8(1); +|ic1 +[1] + +-- 123 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int16); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == int(8); +||mismatched .* int16 .* int64 + +-- 124 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int16); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|jc1 +[1] + +-- 125 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int32); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == int(8); +||mismatched .* int32 .* int64 + +-- 126 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int32); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|kc1 +[1] + +-- 127 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int64); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == byte(8); +||mismatched .* int64 .* uint8 + +-- 128 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int64); + INSERT INTO t VALUES (int64(1)); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|lc1 +[1] + +-- 129 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|lc1 +[1] + +-- 130 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint8); + INSERT INTO t VALUES (byte(1)); +COMMIT; +SELECT * from t +WHERE c1 == int8(8); +||mismatched .* uint8 .* int8 + +-- 131 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint8); + INSERT INTO t VALUES (byte(1)); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|uc1 +[1] + +-- 132 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint16); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == byte(8); +||mismatched .* uint16 .* uint8 + +-- 133 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint16); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|vc1 +[1] + +-- 134 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint32); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|wc1 +[1] + +-- 135 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint64); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == int(8); +||mismatched .* uint64 .* int64 + +-- 136 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint64); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|xc1 +[1] + +-- 137 +BEGIN TRANSACTION; + CREATE TABLE t (c1 uint); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * from t +WHERE c1 == 1; +|xc1 +[1] + +-- 138 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float32); + INSERT INTO t VALUES (8); +COMMIT; +SELECT * from t +WHERE c1 == byte(8); +||mismatched .* float32 .* uint8 + +-- 139 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float32); + INSERT INTO t VALUES (8); +COMMIT; +SELECT * from t +WHERE c1 == 8; +|fc1 +[8] + +-- 140 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float64); + INSERT INTO t VALUES (2); +COMMIT; +SELECT * from t +WHERE c1 == byte(2); +||mismatched .* float64 .* uint8 + +-- 141 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float64); + INSERT INTO t VALUES (2); +COMMIT; +SELECT * from t +WHERE c1 == 2; +|gc1 +[2] + +-- 142 +BEGIN TRANSACTION; + CREATE TABLE t (c1 float); + INSERT INTO t VALUES (2.); +COMMIT; +SELECT * from t +WHERE c1 == 2; +|gc1 +[2] + +-- 143 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex64); + INSERT INTO t VALUES (complex(2., 5.)); +COMMIT; +SELECT * from t +WHERE c1 == "foo"; +||mismatched .* complex64 .* string + +-- 144 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex64); + INSERT INTO t VALUES (complex(2, 5.)); +COMMIT; +SELECT * from t +WHERE c1 == 2+5i; +|cc1 +[(2+5i)] + +-- 145 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex128); + INSERT INTO t VALUES (2+5i); +COMMIT; +SELECT * from t +WHERE c1 == "2"; +||mismatched .* complex128 .* string + +-- 146 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex128); + INSERT INTO t VALUES (2+5i); +COMMIT; +SELECT * from t +WHERE c1 == complex(2, 5); +|dc1 +[(2+5i)] + +-- 147 +BEGIN TRANSACTION; + CREATE TABLE t (c1 string); + INSERT INTO t VALUES ("foo"); +COMMIT; +SELECT * from t +WHERE c1 == 2; +||mismatched .* string .* int64 + +-- 148 +BEGIN TRANSACTION; + CREATE TABLE t (c1 string); + INSERT INTO t VALUES ("f"+"oo"); +COMMIT; +SELECT * from t +WHERE c1 == "fo"+"o"; +|sc1 +[foo] + +-- 149 +SELECT 2/(3*5-15) AS foo FROM bar; +||division by zero + +-- 150 +SELECT 2.0/(2.0-2.0) AS foo FROM bar; +||division by zero + +-- 151 +SELECT 2i/(2i-2i) AS foo FROM bar; +||division by zero + +-- 152 +SELECT 2/(3*5-x) AS foo FROM bar; +||table .* not exist + +-- S 153 +SELECT 314, 42 AS AUQLUE, DepartmentID, DepartmentID+1000, LastName AS Name +FROM employee +ORDER BY Name; +|l, lAUQLUE, lDepartmentID, l, sName +[314 42 33 1033 Heisenberg] +[314 42 John] +[314 42 33 1033 Jones] +[314 42 31 1031 Rafferty] +[314 42 34 1034 Robinson] +[314 42 34 1034 Smith] + +-- S 154 +SELECT * +FROM + employee AS e, + ( SELECT * FROM department) +ORDER BY e.LastName; +|se.LastName, le.DepartmentID, l, s +[Heisenberg 33 35 Marketing] +[Heisenberg 33 34 Clerical] +[Heisenberg 33 33 Engineering] +[Heisenberg 33 31 Sales] +[John 35 Marketing] +[John 34 Clerical] +[John 33 Engineering] +[John 31 Sales] +[Jones 33 35 Marketing] +[Jones 33 34 Clerical] +[Jones 33 33 Engineering] +[Jones 33 31 Sales] +[Rafferty 31 35 Marketing] +[Rafferty 31 34 Clerical] +[Rafferty 31 33 Engineering] +[Rafferty 31 31 Sales] +[Robinson 34 35 Marketing] +[Robinson 34 34 Clerical] +[Robinson 34 33 Engineering] +[Robinson 34 31 Sales] +[Smith 34 35 Marketing] +[Smith 34 34 Clerical] +[Smith 34 33 Engineering] +[Smith 34 31 Sales] + +-- S 155 +SELECT * FROM employee AS e, ( SELECT * FROM department) AS d +ORDER BY e.LastName; +|se.LastName, le.DepartmentID, ld.DepartmentID, sd.DepartmentName +[Heisenberg 33 35 Marketing] +[Heisenberg 33 34 Clerical] +[Heisenberg 33 33 Engineering] +[Heisenberg 33 31 Sales] +[John 35 Marketing] +[John 34 Clerical] +[John 33 Engineering] +[John 31 Sales] +[Jones 33 35 Marketing] +[Jones 33 34 Clerical] +[Jones 33 33 Engineering] +[Jones 33 31 Sales] +[Rafferty 31 35 Marketing] +[Rafferty 31 34 Clerical] +[Rafferty 31 33 Engineering] +[Rafferty 31 31 Sales] +[Robinson 34 35 Marketing] +[Robinson 34 34 Clerical] +[Robinson 34 33 Engineering] +[Robinson 34 31 Sales] +[Smith 34 35 Marketing] +[Smith 34 34 Clerical] +[Smith 34 33 Engineering] +[Smith 34 31 Sales] + +-- 156 +BEGIN TRANSACTION; + CREATE TABLE t (c1 int32, c2 string); + INSERT INTO t VALUES (1, "a"); + INSERT INTO t VALUES (int64(2), "b"); +COMMIT; +SELECT c2 FROM t; +||cannot .*int64.*int32 .* c1 + +-- 157 +BEGIN TRANSACTION; + CREATE TABLE t (c1 complex64); + INSERT INTO t VALUES(1); +COMMIT; +SELECT * FROM t; +|cc1 +[(1+0i)] + +-- 158 +BEGIN TRANSACTION; + CREATE TABLE p (p bool); + INSERT INTO p VALUES (NULL), (false), (true); +COMMIT; +SELECT * FROM p; +|bp +[true] +[false] +[] + +-- 159 +BEGIN TRANSACTION; + CREATE TABLE p (p bool); + INSERT INTO p VALUES (NULL), (false), (true); +COMMIT; +SELECT p.p AS p, q.p AS q, p.p ⩖ q.p AS p_or_q, p.p && q.p aS p_and_q FROM p, p AS q; +|bp, bq, bp_or_q, bp_and_q +[true true true true] +[true false true false] +[true true ] +[false true true false] +[false false false false] +[false false] +[ true true ] +[ false false] +[ ] + +-- 160 +BEGIN TRANSACTION; + CREATE TABLE p (p bool); + INSERT INTO p VALUES (NULL), (false), (true); +COMMIT; +SELECT p, !p AS not_p FROM p; +|bp, bnot_p +[true false] +[false true] +[ ] + +-- S 161 +SELECT * FROM department WHERE DepartmentID >= 33 +ORDER BY DepartmentID; +|lDepartmentID, sDepartmentName +[33 Engineering] +[34 Clerical] +[35 Marketing] + +-- S 162 +SELECT * FROM department WHERE DepartmentID <= 34 +ORDER BY DepartmentID; +|lDepartmentID, sDepartmentName +[31 Sales] +[33 Engineering] +[34 Clerical] + +-- S 163 +SELECT * FROM department WHERE DepartmentID < 34 +ORDER BY DepartmentID; +|lDepartmentID, sDepartmentName +[31 Sales] +[33 Engineering] + +-- S 164 +SELECT +DepartmentID FROM employee; +|? +[] +[34] +[34] +[33] +[33] +[31] + +-- S 165 +SELECT * FROM employee +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 33] +[John ] +[Jones 33] +[Rafferty 31] +[Robinson 34] +[Smith 34] + +-- S 166 +SELECT * +FROM employee +ORDER BY LastName DESC; +|sLastName, lDepartmentID +[Smith 34] +[Robinson 34] +[Rafferty 31] +[Jones 33] +[John ] +[Heisenberg 33] + +-- S 167 +SELECT 1023+DepartmentID AS y FROM employee +ORDER BY y DESC; +|ly +[1057] +[1057] +[1056] +[1056] +[1054] +[] + +-- S 168 +SELECT +DepartmentID AS y FROM employee +ORDER BY y DESC; +|ly +[34] +[34] +[33] +[33] +[31] +[] + +-- S 169 +SELECT * FROM employee ORDER BY DepartmentID, LastName DESC; +|sLastName, lDepartmentID +[Smith 34] +[Robinson 34] +[Jones 33] +[Heisenberg 33] +[Rafferty 31] +[John ] + +-- S 170 +SELECT * FROM employee ORDER BY 0+DepartmentID DESC; +|sLastName, lDepartmentID +[Robinson 34] +[Smith 34] +[Jones 33] +[Heisenberg 33] +[Rafferty 31] +[John ] + +-- S 171 +SELECT * FROM employee ORDER BY +DepartmentID DESC; +|sLastName, lDepartmentID +[Robinson 34] +[Smith 34] +[Jones 33] +[Heisenberg 33] +[Rafferty 31] +[John ] + +-- S 172 +SELECT ^DepartmentID AS y FROM employee +ORDER BY y DESC; +|ly +[-32] +[-34] +[-34] +[-35] +[-35] +[] + +-- S 173 +SELECT ^byte(DepartmentID) AS y FROM employee ORDER BY y DESC; +|uy +[224] +[222] +[222] +[221] +[221] +[] + +-- 174 +BEGIN TRANSACTION; + CREATE TABLE t (r RUNE); + INSERT INTO t VALUES (1), ('A'), (rune(int(0x21))); +COMMIT; +SELECT * FROM t +ORDER BY r; +|kr +[1] +[33] +[65] + +-- 175 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (-2), (-1), (0), (1), (2); +COMMIT; +SELECT i^1 AS y FROM t +ORDER by y; +|ly +[-2] +[-1] +[0] +[1] +[3] + +-- 176 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (-2), (-1), (0), (1), (2); +COMMIT; +SELECT i∨1 AS y FROM t +ORDER BY y; +|ly +[-1] +[-1] +[1] +[1] +[3] + +-- 177 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (-2), (-1), (0), (1), (2); +COMMIT; +SELECT i&1 FROM t; +|l +[0] +[1] +[0] +[1] +[0] + +-- 178 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (-2), (-1), (0), (1), (2); +COMMIT; +SELECT i&^1 AS y FROM t +ORDER BY y; +|ly +[-2] +[-2] +[0] +[0] +[2] + +-- S 179 +SELECT * from employee WHERE LastName == "Jones" ⩖ DepartmentID IS NULL +ORDER by LastName DESC; +|sLastName, lDepartmentID +[Jones 33] +[John ] + +-- S 180 +SELECT * from employee WHERE LastName != "Jones" && DepartmentID IS NOT NULL +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 33] +[Rafferty 31] +[Robinson 34] +[Smith 34] + +-- 181 +SELECT 42[0] FROM foo; +||invalid operation.*index of type + +-- 182 +SELECT "foo"[-1] FROM foo; +||invalid string index.*index .* non.*negative + +-- 183 +SELECT "foo"[3] FROM foo; +||invalid string index.*out of bounds + +-- 184 +SELECT "foo"["bar">true] FROM foo; +||mismatched type + +-- S 185 +SELECT DepartmentID[0] FROM employee; +||run.time error.*invalid operation.*index of type + +-- S 186 +SELECT "foo"[-DepartmentID] FROM employee; +||run.time error.*invalid string index.*index .* non.*negative + +-- S 187 +SELECT LastName[100] FROM employee; +||run.time.error.*invalid string index.*out of bounds + +-- S 188 +SELECT LastName[0], LastName FROM employee ORDER BY LastName; +|u, sLastName +[72 Heisenberg] +[74 John] +[74 Jones] +[82 Rafferty] +[82 Robinson] +[83 Smith] + +-- S 189 +SELECT LastName, string(LastName[0]), string(LastName[1]), string(LastName[2]), string(LastName[3]) +FROM employee +ORDER BY LastName; +|sLastName, s, s, s, s +[Heisenberg H e i s] +[John J o h n] +[Jones J o n e] +[Rafferty R a f f] +[Robinson R o b i] +[Smith S m i t] + +-- S 190 +SELECT LastName, LastName[:], LastName[:2], LastName[2:], LastName[1:3] +FROM employee +ORDER by LastName; +|sLastName, s, s, s, s +[Heisenberg Heisenberg He isenberg ei] +[John John Jo hn oh] +[Jones Jones Jo nes on] +[Rafferty Rafferty Ra fferty af] +[Robinson Robinson Ro binson ob] +[Smith Smith Sm ith mi] + +-- S 191 +SELECT LastName +FROM employee +WHERE department IS NULL; +||unknown field department + +-- S 192 +SELECT + DepartmentID, + LastName, + LastName[:4], + LastName[:0*DepartmentID], + LastName[0*DepartmentID:0], + LastName[0*DepartmentID:0*DepartmentID], +FROM + employee, +ORDER BY LastName DESC; +|lDepartmentID, sLastName, s, s, s, s +[34 Smith Smit ] +[34 Robinson Robi ] +[31 Rafferty Raff ] +[33 Jones Jone ] +[ John John ] +[33 Heisenberg Heis ] + +-- S 193 +SELECT + DepartmentID AS x, + DepartmentID<<1 AS a, + 1<>1 AS a, + uint(1)<<63>>uint(DepartmentID) AS b, +FROM + employee, +WHERE DepartmentID IS NOT NULL +ORDER BY x; +|lx, la, xb +[31 15 4294967296] +[33 16 1073741824] +[33 16 1073741824] +[34 17 536870912] +[34 17 536870912] + +-- S 195 +SELECT DISTINCT DepartmentID +FROM employee +WHERE DepartmentID IS NOT NULL; +|lDepartmentID +[31] +[33] +[34] + +-- S 196 +SELECT DISTINCT e.DepartmentID, d.DepartmentID, e.LastName +FROM employee AS e, department AS d +WHERE e.DepartmentID == d.DepartmentID; +|le.DepartmentID, ld.DepartmentID, se.LastName +[31 31 Rafferty] +[33 33 Heisenberg] +[33 33 Jones] +[34 34 Robinson] +[34 34 Smith] + +-- S 197 +SELECT DISTINCT e.DepartmentID, d.DepartmentID, e.LastName +FROM employee AS e, department AS d +WHERE e.DepartmentID == d.DepartmentID +ORDER BY e.LastName; +|le.DepartmentID, ld.DepartmentID, se.LastName +[33 33 Heisenberg] +[33 33 Jones] +[31 31 Rafferty] +[34 34 Robinson] +[34 34 Smith] + +-- S 198, http://en.wikipedia.org/wiki/Join_(SQL)#Cross_join +SELECT * +FROM employee, department +ORDER BY employee.LastName, department.DepartmentID; +|semployee.LastName, lemployee.DepartmentID, ldepartment.DepartmentID, sdepartment.DepartmentName +[Heisenberg 33 31 Sales] +[Heisenberg 33 33 Engineering] +[Heisenberg 33 34 Clerical] +[Heisenberg 33 35 Marketing] +[John 31 Sales] +[John 33 Engineering] +[John 34 Clerical] +[John 35 Marketing] +[Jones 33 31 Sales] +[Jones 33 33 Engineering] +[Jones 33 34 Clerical] +[Jones 33 35 Marketing] +[Rafferty 31 31 Sales] +[Rafferty 31 33 Engineering] +[Rafferty 31 34 Clerical] +[Rafferty 31 35 Marketing] +[Robinson 34 31 Sales] +[Robinson 34 33 Engineering] +[Robinson 34 34 Clerical] +[Robinson 34 35 Marketing] +[Smith 34 31 Sales] +[Smith 34 33 Engineering] +[Smith 34 34 Clerical] +[Smith 34 35 Marketing] + +-- S 199, http://en.wikipedia.org/wiki/Join_(SQL)#Inner_join +SELECT * +FROM employee, department +WHERE employee.DepartmentID == department.DepartmentID +ORDER BY employee.LastName, department.DepartmentID; +|semployee.LastName, lemployee.DepartmentID, ldepartment.DepartmentID, sdepartment.DepartmentName +[Heisenberg 33 33 Engineering] +[Jones 33 33 Engineering] +[Rafferty 31 31 Sales] +[Robinson 34 34 Clerical] +[Smith 34 34 Clerical] + +-- S 200 +BEGIN TRANSACTION; + INSERT INTO department (DepartmentID, DepartmentName) + SELECT DepartmentID+1000, DepartmentName+"/headquarters" + FROM department; +COMMIT; +SELECT * FROM department +ORDER BY DepartmentID; +|lDepartmentID, sDepartmentName +[31 Sales] +[33 Engineering] +[34 Clerical] +[35 Marketing] +[1031 Sales/headquarters] +[1033 Engineering/headquarters] +[1034 Clerical/headquarters] +[1035 Marketing/headquarters] + +-- S 201` +BEGIN TRANSACTION; + INSERT INTO department (DepartmentName, DepartmentID) + SELECT DepartmentName+"/headquarters", DepartmentID+1000 + FROM department; +COMMIT; +SELECT * FROM department +ORDER BY DepartmentID; +|lDepartmentID, sDepartmentName +[31 Sales] +[33 Engineering] +[34 Clerical] +[35 Marketing] +[1031 Sales/headquarters] +[1033 Engineering/headquarters] +[1034 Clerical/headquarters] +[1035 Marketing/headquarters] + +-- S 202 +BEGIN TRANSACTION; + DELETE FROM department; +COMMIT; +SELECT * FROM department +|?DepartmentID, ?DepartmentName + +-- S 203 +BEGIN TRANSACTION; + DELETE FROM department + WHERE DepartmentID == 35 ⩖ DepartmentName != "" && DepartmentName[0] == 'C'; +COMMIT; +SELECT * FROM department +ORDER BY DepartmentID; +|lDepartmentID, sDepartmentName +[31 Sales] +[33 Engineering] + +-- S 204 +SELECT id(), LastName +FROM employee +ORDER BY id(); +|l, sLastName +[5 Rafferty] +[6 Jones] +[7 Heisenberg] +[8 Robinson] +[9 Smith] +[10 John] + +-- S 205 +BEGIN TRANSACTION; + DELETE FROM employee + WHERE LastName == "Jones"; +COMMIT; +SELECT id(), LastName +FROM employee +ORDER BY id(); +|l, sLastName +[5 Rafferty] +[7 Heisenberg] +[8 Robinson] +[9 Smith] +[10 John] + +-- S 206 +BEGIN TRANSACTION; + DELETE FROM employee + WHERE LastName == "Jones"; + INSERT INTO employee (LastName) VALUES ("Jones"); +COMMIT; +SELECT id(), LastName +FROM employee +ORDER BY id(); +|l, sLastName +[5 Rafferty] +[7 Heisenberg] +[8 Robinson] +[9 Smith] +[10 John] +[11 Jones] + +-- S 207 +SELECT id(), e.LastName, e.DepartmentID, d.DepartmentID +FROM + employee AS e, + department AS d, +WHERE e.DepartmentID == d.DepartmentID +ORDER BY e.LastName; +|?, se.LastName, le.DepartmentID, ld.DepartmentID +[ Heisenberg 33 33] +[ Jones 33 33] +[ Rafferty 31 31] +[ Robinson 34 34] +[ Smith 34 34] + +-- S 208 +SELECT e.ID, e.LastName, e.DepartmentID, d.DepartmentID +FROM + (SELECT id() AS ID, LastName, DepartmentID FROM employee;) AS e, + department AS d, +WHERE e.DepartmentID == d.DepartmentID +ORDER BY e.ID; +|le.ID, se.LastName, le.DepartmentID, ld.DepartmentID +[5 Rafferty 31 31] +[6 Jones 33 33] +[7 Heisenberg 33 33] +[8 Robinson 34 34] +[9 Smith 34 34] + +-- S 209 +BEGIN TRANSACTION; + UPDATE none + DepartmentID = DepartmentID+1000, + WHERE DepartmentID == 33; +COMMIT; +SELECT * FROM employee; +||table.*not.*exist + +-- S 210 +BEGIN TRANSACTION; + UPDATE employee + FirstName = "John" + WHERE DepartmentID == 33; +COMMIT; +SELECT * FROM employee; +||unknown.*FirstName + +-- S 211 +BEGIN TRANSACTION; + UPDATE employee + DepartmentID = DepartmentID+1000, + WHERE DepartmentID == 33; +COMMIT; +SELECT * FROM employee +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 1033] +[John ] +[Jones 1033] +[Rafferty 31] +[Robinson 34] +[Smith 34] + +-- S 212 +BEGIN TRANSACTION; + UPDATE employee + DepartmentID = DepartmentID+1000, + LastName = "Mr. "+LastName + WHERE id() == 7; +COMMIT; +SELECT * FROM employee +ORDER BY LastName DESC; +|sLastName, lDepartmentID +[Smith 34] +[Robinson 34] +[Rafferty 31] +[Mr. Heisenberg 1033] +[Jones 33] +[John ] + +-- S 213 +BEGIN TRANSACTION; + UPDATE employee + LastName = "Mr. "+LastName, + DepartmentID = DepartmentID+1000, + WHERE id() == 7; +COMMIT; +SELECT * FROM employee +ORDER BY LastName DESC; +|sLastName, lDepartmentID +[Smith 34] +[Robinson 34] +[Rafferty 31] +[Mr. Heisenberg 1033] +[Jones 33] +[John ] + +-- S 214 +BEGIN TRANSACTION; + UPDATE employee + DepartmentID = DepartmentID+1000; +COMMIT; +SELECT * FROM employee +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 1033] +[John ] +[Jones 1033] +[Rafferty 1031] +[Robinson 1034] +[Smith 1034] + +-- S 215 +BEGIN TRANSACTION; + UPDATE employee + DepartmentId = DepartmentID+1000; +COMMIT; +SELECT * FROM employee; +||unknown + +-- S 216 +BEGIN TRANSACTION; + UPDATE employee + DepartmentID = DepartmentId+1000; +COMMIT; +SELECT * FROM employee; +||unknown + +-- S 217 +BEGIN TRANSACTION; + UPDATE employee + DepartmentID = "foo"; +COMMIT; +SELECT * FROM employee; +||cannot .* string.*int64 .* DepartmentID + +-- S 218 +SELECT foo[len()] FROM bar; +||missing argument + +-- S 219 +SELECT foo[len(42)] FROM bar; +||invalid argument + +-- S 220 +SELECT foo[len(42, 24)] FROM bar; +||too many + +-- S 221 +SELECT foo[len("baz")] FROM bar; +||table + +-- S 222 +SELECT LastName[len("baz")-4] FROM employee; +||invalid string index + +-- S 223 +SELECT LastName[:len(LastName)-3] AS y FROM employee +ORDER BY y; +|sy +[Heisenb] +[J] +[Jo] +[Raffe] +[Robin] +[Sm] + +-- S 224 +SELECT complex(float32(DepartmentID+int(id())), 0) AS x, complex(DepartmentID+int(id()), 0) +FROM employee +ORDER by real(x) DESC; +|cx, d +[(43+0i) (43+0i)] +[(42+0i) (42+0i)] +[(40+0i) (40+0i)] +[(39+0i) (39+0i)] +[(36+0i) (36+0i)] +[ ] + +-- S 225 +SELECT real(complex(float32(DepartmentID+int(id())), 0)) AS x, real(complex(DepartmentID+int(id()), 0)) +FROM employee +ORDER BY x DESC; +|fx, g +[43 43] +[42 42] +[40 40] +[39 39] +[36 36] +[ ] + +-- S 226 +SELECT imag(complex(0, float32(DepartmentID+int(id())))) AS x, imag(complex(0, DepartmentID+int(id()))) +FROM employee +ORDER BY x DESC; +|fx, g +[43 43] +[42 42] +[40 40] +[39 39] +[36 36] +[ ] + +-- 227 +BEGIN TRANSACTION; + CREATE TABLE t (c string); + INSERT INTO t VALUES("foo"), ("bar"); + DELETE FROM t WHERE c == "foo"; +COMMIT; +SELECT 100*id(), c FROM t; +|l, sc +[200 bar] + +-- 228 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE a; +COMMIT; +SELECT * FROM a; +||table a does not exist + +-- 229 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE a; +COMMIT; +SELECT * FROM b; +|?b + +-- 230 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE a; +COMMIT; +SELECT * FROM c; +|?c + +-- 231 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE b; +COMMIT; +SELECT * FROM a; +|?a + +-- 232 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE b; +COMMIT; +SELECT * FROM b; +||table b does not exist + +-- 233 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE b; +COMMIT; +SELECT * FROM c; +|?c + +-- 234 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE c; +COMMIT; +SELECT * FROM a; +|?a + +-- 235 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE c; +COMMIT; +SELECT * FROM b; +|?b + +-- 236 +BEGIN TRANSACTION; + CREATE TABLE a (a int); + CREATE TABLE b (b int); + CREATE TABLE c (c int); + DROP TABLE c; +COMMIT; +SELECT * FROM c; +||table c does not exist + +-- 237 +BEGIN TRANSACTION; + CREATE TABLE a (c int); + INSERT INTO a VALUES (10), (11), (12); + CREATE TABLE b (d int); + INSERT INTO b VALUES (20), (21), (22), (23); +COMMIT; +SELECT * FROM a, b; +|la.c, lb.d +[12 23] +[12 22] +[12 21] +[12 20] +[11 23] +[11 22] +[11 21] +[11 20] +[10 23] +[10 22] +[10 21] +[10 20] + +-- 238 +BEGIN TRANSACTION; + CREATE TABLE a (c int); + INSERT INTO a VALUES (0), (1), (2); +COMMIT; +SELECT + 9*x2.c AS x2, + 3*x1.c AS x1, + 1*x0.c AS x0, + 9*x2.c + 3*x1.c + x0.c AS y, +FROM + a AS x2, + a AS x1, + a AS x0, +ORDER BY y; +|lx2, lx1, lx0, ly +[0 0 0 0] +[0 0 1 1] +[0 0 2 2] +[0 3 0 3] +[0 3 1 4] +[0 3 2 5] +[0 6 0 6] +[0 6 1 7] +[0 6 2 8] +[9 0 0 9] +[9 0 1 10] +[9 0 2 11] +[9 3 0 12] +[9 3 1 13] +[9 3 2 14] +[9 6 0 15] +[9 6 1 16] +[9 6 2 17] +[18 0 0 18] +[18 0 1 19] +[18 0 2 20] +[18 3 0 21] +[18 3 1 22] +[18 3 2 23] +[18 6 0 24] +[18 6 1 25] +[18 6 2 26] + +-- 239 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + INSERT INTO t VALUES (242); + DELETE FROM t WHERE c != 0; +COMMIT; +SELECT * FROM t +|?c + +-- 240 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (242), (12, 24); +COMMIT; +||expect + +-- 241 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (24, 2), (1224); +COMMIT; +||expect + +-- 242 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +ROLLBACK; +SELECT * from t; +||does not exist + +-- 243 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +BEGIN TRANSACTION; + DROP TABLE T; +COMMIT; +SELECT * from t; +||does not exist + +-- 244 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +BEGIN TRANSACTION; + DROP TABLE t; +ROLLBACK; +SELECT * from t; +|?i + +-- 245 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (int(1.2)); +COMMIT; +SELECT * FROM t; +||truncated + +-- 246 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (string(65.0)); +COMMIT; +SELECT * FROM t; +||cannot convert + +-- 247 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES (string(65)); +COMMIT; +SELECT * FROM t; +|ss +[A] + +-- 248 +BEGIN TRANSACTION; + CREATE TABLE t (i uint32); + INSERT INTO t VALUES (uint32(int8(uint16(0x10F0)))); +COMMIT; +SELECT i == 0xFFFFFFF0 FROM t; +|b +[true] + +-- 249 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + (string('a')), // "a" + (string(-1)), // "\ufffd" == "\xef\xbf\xbd" + (string(0xf8)), // "\u00f8" == "ø" == "\xc3\xb8" + (string(0x65e5)), // "\u65e5" == "日" == "\xe6\x97\xa5" + ; +COMMIT; +SELECT + id() == 1 && s == "a" ⩖ + id() == 2 && s == "\ufffd" && s == "\xef\xbf\xbd" ⩖ + id() == 3 && s == "\u00f8" && s == "ø" && s == "\xc3\xb8" ⩖ + id() == 4 && s == "\u65e5" && s == "日" && s == "\xe6\x97\xa5" +FROM t; +|b +[true] +[true] +[true] +[true] + +-- 250 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (0); +COMMIT; +SELECT 2.3+1, 1+2.3 FROM t; +|f, f +[3.3 3.3] + +-- 251 +BEGIN TRANSACTION; + CREATE TABLE t (i byte); + INSERT INTO t VALUES (-1+byte(2)); +COMMIT; +SELECT * FROM t; +||mismatched + +-- 252 +BEGIN TRANSACTION; + CREATE TABLE t (i byte); + INSERT INTO t VALUES (1+byte(2)); +COMMIT; +SELECT * FROM t; +|ui +[3] + +-- 253 +BEGIN TRANSACTION; + CREATE TABLE t (i byte); + INSERT INTO t VALUES (255+byte(2)); +COMMIT; +SELECT * FROM t; +|ui +[1] + +-- 254 +BEGIN TRANSACTION; + CREATE TABLE t (i byte); + INSERT INTO t VALUES (256+byte(2)); +COMMIT; +SELECT * FROM t; +||mismatched + +-- 255 +BEGIN TRANSACTION; + CREATE TABLE t (i int8); + INSERT INTO t VALUES (127+int8(2)); +COMMIT; +SELECT * FROM t; +|ii +[-127] + +-- 256 +BEGIN TRANSACTION; + CREATE TABLE t (i int8); + INSERT INTO t VALUES (-129+int8(2)); +COMMIT; +SELECT * FROM t; +||mismatched + +-- 257 +BEGIN TRANSACTION; + CREATE TABLE t (i int8); + INSERT INTO t VALUES (-128+int8(2)); +COMMIT; +SELECT * FROM t; +|ii +[-126] + +-- 258 +BEGIN TRANSACTION; + CREATE TABLE t (i int8); + INSERT INTO t VALUES (128+int8(2)); +COMMIT; +SELECT * FROM t; +||mismatched + +-- S 259 +SELECT count(none) FROM employee; +||unknown + +-- S 260 +SELECT count() FROM employee; +|l +[6] + +-- S 261 +SELECT count() AS y FROM employee; +|ly +[6] + +-- S 262 +SELECT 3*count() AS y FROM employee; +|ly +[18] + +-- S 263 +SELECT count(LastName) FROM employee; +|l +[6] + +-- S 264 +SELECT count(DepartmentID) FROM employee; +|l +[5] + +-- S 265 +SELECT count() - count(DepartmentID) FROM employee; +|l +[1] + +-- S 266 +SELECT min(LastName), min(DepartmentID) FROM employee; +|s, l +[Heisenberg 31] + +-- S 267 +SELECT max(LastName), max(DepartmentID) FROM employee; +|s, l +[Smith 34] + +-- S 268 +SELECT sum(LastName), sum(DepartmentID) FROM employee; +||cannot + +-- S 269 +SELECT sum(DepartmentID) FROM employee; +|l +[165] + +-- S 270 +SELECT avg(DepartmentID) FROM employee; +|l +[33] + +-- S 271 +SELECT DepartmentID FROM employee GROUP BY none; +||unknown + +-- S 272 +SELECT DepartmentID, sum(DepartmentID) AS s FROM employee GROUP BY DepartmentID ORDER BY s DESC; +|lDepartmentID, ls +[34 68] +[33 66] +[31 31] +[ ] + +-- S 273 +SELECT DepartmentID, count(LastName+string(DepartmentID)) AS y FROM employee GROUP BY DepartmentID ORDER BY y DESC ; +|lDepartmentID, ly +[34 2] +[33 2] +[31 1] +[ 0] + +-- S 274 +SELECT DepartmentID, sum(2*DepartmentID) AS s FROM employee GROUP BY DepartmentID ORDER BY s DESC; +|lDepartmentID, ls +[34 136] +[33 132] +[31 62] +[ ] + +-- S 275 +SELECT min(2*DepartmentID) FROM employee; +|l +[62] + +-- S 276 +SELECT max(2*DepartmentID) FROM employee; +|l +[68] + +-- S 277 +SELECT avg(2*DepartmentID) FROM employee; +|l +[66] + +-- S 278 +SELECT * FROM employee GROUP BY DepartmentID; +|sLastName, ?DepartmentID +[John ] +[Rafferty 31] +[Heisenberg 33] +[Smith 34] + +-- S 279 +SELECT * FROM employee GROUP BY DepartmentID ORDER BY LastName DESC; +|sLastName, lDepartmentID +[Smith 34] +[Rafferty 31] +[John ] +[Heisenberg 33] + +-- S 280 +SELECT * FROM employee GROUP BY DepartmentID, LastName ORDER BY LastName DESC; +|sLastName, lDepartmentID +[Smith 34] +[Robinson 34] +[Rafferty 31] +[Jones 33] +[John ] +[Heisenberg 33] + +-- S 281 +SELECT * FROM employee GROUP BY LastName, DepartmentID ORDER BY LastName DESC; +|sLastName, lDepartmentID +[Smith 34] +[Robinson 34] +[Rafferty 31] +[Jones 33] +[John ] +[Heisenberg 33] + +-- 282 +BEGIN TRANSACTION; + CREATE TABLE s (i int); + CREATE TABLE t (i int); +COMMIT; +BEGIN TRANSACTION; + DROP TABLE s; +COMMIT; +SELECT * FROM t; +|?i + +-- 283 +BEGIN TRANSACTION; + CREATE TABLE t (n int); +COMMIT; +SELECT count() FROM t; +|l +[0] + +-- 284 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT count() FROM t; +|l +[2] + +-- 285 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT count() FROM t WHERE n < 2; +|l +[2] + +-- 286 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT count() FROM t WHERE n < 1; +|l +[1] + +-- 287 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT count() FROM t WHERE n < 0; +|l +[0] + +-- 288 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT s+10 FROM (SELECT sum(n) AS s FROM t WHERE n < 2); +|l +[11] + +-- 289 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT s+10 FROM (SELECT sum(n) AS s FROM t WHERE n < 1); +|l +[10] + +-- 290 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT s+10 FROM (SELECT sum(n) AS s FROM t WHERE n < 0); +|? +[] + +-- 291 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT sum(n) AS s FROM t WHERE n < 2; +|ls +[1] + +-- 292 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT sum(n) AS s FROM t WHERE n < 1; +|ls +[0] + +-- 293 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1); +COMMIT; +SELECT sum(n) AS s FROM t WHERE n < 0; +|?s +[] + +-- 294 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t SELECT count() FROM t; + INSERT INTO t SELECT count() FROM t; + INSERT INTO t SELECT count() FROM t; +COMMIT; +SELECT count() FROM t; +|l +[3] + +-- 295 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t SELECT count() FROM t; + INSERT INTO t SELECT count() FROM t; + INSERT INTO t SELECT count() FROM t; + INSERT INTO t SELECT * FROM t; +COMMIT; +SELECT count() FROM t; +|l +[6] + +-- 296 +BEGIN TRANSACTION; + CREATE TABLE t (n int); + INSERT INTO t VALUES (0), (1), (2); + INSERT INTO t SELECT * FROM t; +COMMIT; +SELECT count() FROM t; +|l +[6] + +-- 297 +BEGIN TRANSACTION; + CREATE TABLE t(S string); + INSERT INTO t SELECT "perfect!" FROM (SELECT count() AS cnt FROM t WHERE S == "perfect!") WHERE cnt == 0; +COMMIT; +SELECT count() FROM t; +|l +[1] + +-- 298 +BEGIN TRANSACTION; + CREATE TABLE t(S string); + INSERT INTO t SELECT "perfect!" FROM (SELECT count() AS cnt FROM t WHERE S == "perfect!") WHERE cnt == 0; + INSERT INTO t SELECT "perfect!" FROM (SELECT count() AS cnt FROM t WHERE S == "perfect!") WHERE cnt == 0; +COMMIT; +SELECT count() FROM t; +|l +[1] + +-- 299 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob("a")); +COMMIT; +SELECT * FROM t; +|?c +[[97]] + +-- 300 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob(` +0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +`)); +COMMIT; +SELECT * FROM t; +|?c +[[10 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 10]] + +-- 301 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob( +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF")); +COMMIT; +SELECT * FROM t; +|?c +[[48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 65 66 67 68 69 70]] + +-- 302 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob( +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF"+ +"!")); +COMMIT; +SELECT * FROM t; +|?c +[[48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 65 66 67 68 69 70 33]] + +-- 303 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob("hell\xc3\xb8")); +COMMIT; +SELECT string(c) FROM t; +|s +[hellø] + +-- 304 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob( +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ +"0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF"+ +"!")); +COMMIT; +SELECT string(c) FROM t; +|s +[0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] + +-- 305 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob("")); +COMMIT; +SELECT string(c) FROM t; +|s +[] + +-- 306 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob("hellø")); +COMMIT; +SELECT * FROM t; +|?c +[[104 101 108 108 195 184]] + +-- 307 +BEGIN TRANSACTION; + CREATE TABLE t(c blob); + INSERT INTO t VALUES (blob("")); +COMMIT; +SELECT * FROM t; +|?c +[[]] + +-- 308 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + ; +COMMIT; +SELECT * FROM t; +|li, ?b +[0 [48]] + +-- 309 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + (1, blob("1")), + ; +COMMIT; +SELECT * FROM t; +|li, ?b +[1 [49]] +[0 [48]] + +-- 310 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + (1, blob("1")), + (2, blob("2")), + ; +COMMIT; +SELECT * FROM t; +|li, ?b +[2 [50]] +[1 [49]] +[0 [48]] + +-- 311 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + ; + DELETE FROM t WHERE i == 0; +COMMIT; +SELECT * FROM t; +|?i, ?b + +-- 312 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + (1, blob("1")), + ; + DELETE FROM t WHERE i == 0; +COMMIT; +SELECT * FROM t; +|li, ?b +[1 [49]] + +-- 313 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + (1, blob("1")), + ; + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t; +|li, ?b +[0 [48]] + +-- 314 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + (1, blob("1")), + (2, blob("2")), + ; + DELETE FROM t WHERE i == 0; +COMMIT; +SELECT * FROM t; +|li, ?b +[2 [50]] +[1 [49]] + +-- 315 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + (1, blob("1")), + (2, blob("2")), + ; + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t; +|li, ?b +[2 [50]] +[0 [48]] + +-- 316 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob("0")), + (1, blob("1")), + (2, blob("2")), + ; + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT * FROM t; +|li, ?b +[1 [49]] +[0 [48]] + +-- 317 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + ; + DELETE FROM t WHERE i == 0; +COMMIT; +SELECT i, string(b) FROM t; +|?i, ? + +-- 318 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (1, blob( + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + ; + DELETE FROM t WHERE i == 0; +COMMIT; +SELECT i, string(b) FROM t; +|li, s +[1 1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] + +-- 319 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (1, blob( + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + ; + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT i, string(b) FROM t; +|li, s +[0 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] + +-- 320 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (1, blob( + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (2, blob( + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + ; + DELETE FROM t WHERE i == 0; +COMMIT; +SELECT i, string(b) FROM t; +|li, s +[2 2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef2123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] +[1 1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] + +-- 321 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (1, blob( + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (2, blob( + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + ; + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT i, string(b) FROM t; +|li, s +[2 2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef2123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] +[0 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] + +-- 322 +BEGIN TRANSACTION; + CREATE TABLE t(i int, b blob); + INSERT INTO t VALUES + (0, blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (1, blob( + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + (2, blob( + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!")), + ; + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT i, string(b) FROM t; +|li, s +[1 1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] +[0 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!] + +-- 323 +BEGIN TRANSACTION; + CREATE TABLE t (c bool); + INSERT INTO t VALUES (false), (true); +COMMIT; +SELECT * FROM t ORDER BY true, c, false; +||cannot .* bool + +-- 324 +BEGIN TRANSACTION; + CREATE TABLE t (c bool, i int); + INSERT INTO t VALUES (false, 1), (true, 2), (false, 10), (true, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|bc, l +[false 11] +[true 22] + +-- 325 +BEGIN TRANSACTION; + CREATE TABLE t (c int8); + INSERT INTO t VALUES (1), (2); +COMMIT; +SELECT * FROM t ORDER BY 42, c, 24; +|ic +[1] +[2] + +-- 326 +BEGIN TRANSACTION; + CREATE TABLE t (c int8, i int); + INSERT INTO t VALUES (99, 1), (100, 2), (99, 10), (100, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|ic, l +[99 11] +[100 22] + +-- 327 +BEGIN TRANSACTION; + CREATE TABLE t (c blob); + INSERT INTO t VALUES (blob("A")), (blob("B")); +COMMIT; +SELECT * FROM t ORDER BY 42, c, 24; +||cannot .* \[\]uint8 + +-- 328 +BEGIN TRANSACTION; + CREATE TABLE t (c blob, i int); + INSERT INTO t VALUES (blob("A"), 1), (blob("B"), 2); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|?c, l +[[65] 1] +[[66] 2] + +-- 329 +BEGIN TRANSACTION; + CREATE TABLE t (c blob, i int); + INSERT INTO t VALUES (blob("A"), 10), (blob("B"), 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|?c, l +[[65] 10] +[[66] 20] + +-- 330 +BEGIN TRANSACTION; + CREATE TABLE t (c blob, i int); + INSERT INTO t VALUES (blob("A"), 1), (blob("B"), 2), (blob("A"), 10), (blob("B"), 20); +COMMIT; +SELECT * FROM t; +|?c, li +[[66] 20] +[[65] 10] +[[66] 2] +[[65] 1] + +-- 331 +BEGIN TRANSACTION; + CREATE TABLE t (c string, i int); + INSERT INTO t VALUES ("A", 1), ("B", 2), ("A", 10), ("B", 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|sc, l +[A 11] +[B 22] + +-- 332 +BEGIN TRANSACTION; + CREATE TABLE t (c blob, i int); + INSERT INTO t VALUES (blob("A"), 1), (blob("B"), 2), (blob("A"), 10), (blob("B"), 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|?c, l +[[65] 11] +[[66] 22] + +-- 333 +BEGIN TRANSACTION; + CREATE TABLE t (c byte); + INSERT INTO t VALUES (42), (314); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +||overflow + +-- 334 +BEGIN TRANSACTION; + CREATE TABLE t (c byte); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|uc +[42] +[114] + +-- 335 +BEGIN TRANSACTION; + CREATE TABLE t (c byte, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|uc, l +[100 11] +[101 22] + +-- 336 +BEGIN TRANSACTION; + CREATE TABLE t (c byte); + INSERT INTO t VALUES (42), (3.14); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +||truncated + +-- 337 +BEGIN TRANSACTION; + CREATE TABLE t (c complex64); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +||cannot order by + +-- 338 +BEGIN TRANSACTION; + CREATE TABLE t (c complex64, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|cc, l +[(100+0i) 11] +[(101+0i) 22] + +-- 339 +BEGIN TRANSACTION; + CREATE TABLE t (c complex128); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +||cannot order by + +-- 340 +BEGIN TRANSACTION; + CREATE TABLE t (c complex128, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|dc, l +[(100+0i) 11] +[(101+0i) 22] + +-- 341 +BEGIN TRANSACTION; + CREATE TABLE t (c float); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|gc +[42] +[114] + +-- 342 +BEGIN TRANSACTION; + CREATE TABLE t (c float, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|gc, l +[100 11] +[101 22] + +-- 343 +BEGIN TRANSACTION; + CREATE TABLE t (c float64); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|gc +[42] +[114] + +-- 344 +BEGIN TRANSACTION; + CREATE TABLE t (c float64, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|gc, l +[100 11] +[101 22] + +-- 345 +BEGIN TRANSACTION; + CREATE TABLE t (c float32); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|fc +[42] +[114] + +-- 346 +BEGIN TRANSACTION; + CREATE TABLE t (c float32, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|fc, l +[100 11] +[101 22] + +-- 347 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|lc +[42] +[114] + +-- 348 +BEGIN TRANSACTION; + CREATE TABLE t (c int, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|lc, l +[100 11] +[101 22] + +-- 349 +BEGIN TRANSACTION; + CREATE TABLE t (c int64); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|lc +[42] +[114] + +-- 350 +BEGIN TRANSACTION; + CREATE TABLE t (c int64, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|lc, l +[100 11] +[101 22] + +-- 351 +BEGIN TRANSACTION; + CREATE TABLE t (c int8); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|ic +[42] +[114] + +-- 352 +BEGIN TRANSACTION; + CREATE TABLE t (c int8, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|ic, l +[100 11] +[101 22] + +-- 353 +BEGIN TRANSACTION; + CREATE TABLE t (c int16); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|jc +[42] +[114] + +-- 354 +BEGIN TRANSACTION; + CREATE TABLE t (c int16, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|jc, l +[100 11] +[101 22] + +-- 355 +BEGIN TRANSACTION; + CREATE TABLE t (c int32); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|kc +[42] +[114] + +-- 356 +BEGIN TRANSACTION; + CREATE TABLE t (c int32, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|kc, l +[100 11] +[101 22] + +-- 357 +BEGIN TRANSACTION; + CREATE TABLE t (c uint); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|xc +[42] +[114] + +-- 358 +BEGIN TRANSACTION; + CREATE TABLE t (c uint, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|xc, l +[100 11] +[101 22] + +-- 359 +BEGIN TRANSACTION; + CREATE TABLE t (c uint64); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|xc +[42] +[114] + +-- 360 +BEGIN TRANSACTION; + CREATE TABLE t (c uint64, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|xc, l +[100 11] +[101 22] + +-- 361 +BEGIN TRANSACTION; + CREATE TABLE t (c uint8); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|uc +[42] +[114] + +-- 362 +BEGIN TRANSACTION; + CREATE TABLE t (c uint8, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|uc, l +[100 11] +[101 22] + +-- 363 +BEGIN TRANSACTION; + CREATE TABLE t (c uint16); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|vc +[42] +[114] + +-- 364 +BEGIN TRANSACTION; + CREATE TABLE t (c uint16, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|vc, l +[100 11] +[101 22] + +-- 365 +BEGIN TRANSACTION; + CREATE TABLE t (c uint32); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|wc +[42] +[114] + +-- 366 +BEGIN TRANSACTION; + CREATE TABLE t (c uint32, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|wc, l +[100 11] +[101 22] + +-- 367 +BEGIN TRANSACTION; + CREATE TABLE t (c blob, i int); + INSERT INTO t VALUES (blob("A"), 1), (blob("B"), 2); + UPDATE t c = blob("C") WHERE i == 2; +COMMIT; +SELECT * FROM t; +|?c, li +[[67] 2] +[[65] 1] + +-- 368 +BEGIN TRANSACTION; + CREATE TABLE t (c blob, i int); + INSERT INTO t VALUES + (blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!" + ), 1), + (blob( + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!" + ), 2); + UPDATE t c = blob( + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "2123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!" + ) WHERE i == 2; +COMMIT; +SELECT * FROM t; +|?c, li +[[50 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 50 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 50 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 50 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 65 66 67 68 69 70 33] 2] +[[48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 65 66 67 68 69 70 33] 1] + +-- 369 +BEGIN TRANSACTION; + CREATE TABLE t (c blob, i int); + INSERT INTO t VALUES + (blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!" + ), 1), + (blob( + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "1123456789abcdef0123456789abcdef0123456789abcdef0123456789ABCDEF!" + ), 2); + UPDATE t i = 42 WHERE i == 2; +COMMIT; +SELECT * FROM t; +|?c, li +[[49 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 49 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 49 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 49 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 65 66 67 68 69 70 33] 42] +[[48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 97 98 99 100 101 102 48 49 50 51 52 53 54 55 56 57 65 66 67 68 69 70 33] 1] + +-- 370 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * FROM t; +|?i +[1] + +-- 371 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint("1")); +COMMIT; +SELECT * FROM t; +|?i +[1] + +-- 372 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint("12345678901234567890123456789")); +COMMIT; +SELECT * FROM t; +|?i +[12345678901234567890123456789] + +-- 373 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint(2e10)); +COMMIT; +SELECT * FROM t; +|?i +[20000000000] + +-- 374 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint(2e18)); +COMMIT; +SELECT * FROM t; +|?i +[2000000000000000000] + +-- 375 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint(2e19)); +COMMIT; +SELECT * FROM t; +|?i +[20000000000000000000] + +-- 376 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint(2e20)); +COMMIT; +SELECT * FROM t; +|?i +[200000000000000000000] + +-- 377 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint("0x1fffffffffffffff")); +COMMIT; +SELECT * FROM t; +|?i +[2305843009213693951] + +-- 378 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + INSERT INTO t VALUES (bigint("0x1ffffffffffffffffffffff")); +COMMIT; +SELECT * FROM t; +|?i +[618970019642690137449562111] + +-- 379 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|?c +[42] +[114] + +-- 380 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|?c, l +[100 11] +[101 22] + +-- 381 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT * FROM t WHERE c > 100 ORDER BY c DESC; +|?c +[111] +[110] +[101] + +-- 382 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT * FROM t WHERE c < 110 ORDER BY c; +|?c +[100] +[101] + +-- 383 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT * FROM t WHERE c <= 110 ORDER BY c; +|?c +[100] +[101] +[110] + +-- 384 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT * FROM t WHERE c >= 110 ORDER BY c; +|?c +[110] +[111] + +-- 385 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT * FROM t WHERE c != 110 ORDER BY c; +|?c +[100] +[101] +[111] + +-- 386 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT * FROM t WHERE c == 110 ORDER BY c; +|?c +[110] + +-- 387 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT (c+1000) as s FROM t ORDER BY s; +|?s +[1100] +[1101] +[1110] +[1111] + +-- 388 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT (1000-c) as s FROM t ORDER BY s; +|?s +[889] +[890] +[899] +[900] + +-- 389 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT (c>>1) as s FROM t ORDER BY s; +|?s +[50] +[50] +[55] +[55] + +-- 390 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (100), (101), (110), (111); +COMMIT; +SELECT (c<<1) as s FROM t ORDER BY s; +|?s +[200] +[202] +[220] +[222] + +-- 391 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE c&0x55555 == 0x11550; +|?c +[79856] + +-- 392 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE c∨0x55555 == 0x577f5; +|?c +[79856] + +-- 393 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE c&^0x55555 == 0x022a0; +|?c +[79856] + +-- 394 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE c^0x55555 == 0x462a5; +|?c +[79856] + +-- 395 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE c%256 == 0xf0; +|?c +[79856] + +-- 396 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE c*16 == 0x137f00; +|?c +[79856] + +-- 397 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE ^c == -(0x137f0+1); +|?c +[79856] + +-- 398 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE +c == 0x137f0; +|?c +[79856] + +-- 399 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint); + INSERT INTO t VALUES (0x137f0); +COMMIT; +SELECT * FROM t WHERE -c == -79856; +|?c +[79856] + +-- 400 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat); + INSERT INTO t VALUES (42), (114); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|?c +[42/1] +[114/1] + +-- 401 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, i int); + INSERT INTO t VALUES (100, 1), (101, 2), (100, 10), (101, 20); +COMMIT; +SELECT c, sum(i) FROM t GROUP BY c; +|?c, l +[100/1 11] +[101/1 22] + +-- 402 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat); + INSERT INTO t VALUES (42.24), (114e3); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|?c +[5944751508129055/140737488355328] +[114000/1] + +-- 403 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat); + INSERT INTO t VALUES ('A'), ('B'); +COMMIT; +SELECT * FROM t ORDER BY 15, c, 16; +|?c +[65/1] +[66/1] + +-- 404 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat); + INSERT INTO t VALUES (bigrat("2/3")+bigrat("5/7")); +COMMIT; +SELECT * FROM t; +|?c +[29/21] + +-- 405 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat); + INSERT INTO t VALUES (bigrat("2/3")-bigrat("5/7")); +COMMIT; +SELECT * FROM t; +|?c +[-1/21] + +-- 406 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat); + INSERT INTO t VALUES (bigrat("2/3")*bigrat("5/7")); +COMMIT; +SELECT * FROM t; +|?c +[10/21] + +-- 407 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint, d bigint); + INSERT INTO t VALUES (1, 0); +COMMIT; +SELECT c/d FROM t; +||division .* zero + +-- 408 +BEGIN TRANSACTION; + CREATE TABLE t (c bigint, d bigint); + INSERT INTO t VALUES (1, 0); +COMMIT; +SELECT c%d FROM t; +||division .* zero + +-- 409 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT c == d FROM t; +|b +[false] + +-- 410 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("4/6")); +COMMIT; +SELECT c == d FROM t; +|b +[true] + +-- 411 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT c != d FROM t; +|b +[true] + +-- 412 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("4/6")); +COMMIT; +SELECT c != d FROM t; +|b +[false] + +-- 413 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT c < d FROM t; +|b +[true] + +-- 414 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("4/6")); +COMMIT; +SELECT c < d FROM t; +|b +[false] + +-- 415 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT c <= d FROM t; +|b +[true] + +-- 416 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("4/6")); +COMMIT; +SELECT c <= d FROM t; +|b +[true] + +-- 417 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT c > d FROM t; +|b +[false] + +-- 418 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("4/6")); +COMMIT; +SELECT c > d FROM t; +|b +[false] + +-- 419 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT c >= d FROM t; +|b +[false] + +-- 420 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("4/6")); +COMMIT; +SELECT c >= d FROM t; +|b +[true] + +-- 421 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT c / d FROM t; +|? +[14/15] + +-- 422 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("0")); +COMMIT; +SELECT c / d FROM t; +||division .* zero + +-- 423 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("0")); +COMMIT; +SELECT c / (6-2*3) FROM t; +||division .* zero + +-- 424 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT +c, -d FROM t; +|?, ? +[2/3 -5/7] + +-- 425 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat, d bigrat); + INSERT INTO t VALUES (bigrat("2/3"), bigrat("5/7")); +COMMIT; +SELECT 1+c, d+1, 1.5+c, d+1.5 FROM t; +|?, ?, ?, ? +[5/3 12/7 13/6 31/14] + +-- 426 +BEGIN TRANSACTION; + CREATE TABLE t (c bigrat); + INSERT INTO t VALUES (bigrat("355/113")); +COMMIT; +SELECT float(c) FROM t; +|g +[3.1415929203539825] + +-- 427 +BEGIN TRANSACTION; + CREATE TABLE t (c time); + INSERT INTO t VALUES (date(2006, 1, 2, 15, 4, 5, 999999999, "CET")); +COMMIT; +SELECT formatTime(c, "2006-01-02 15:04:05.999999999 -0700") FROM t; +|s +[2006-01-02 15:04:05.999999999 +0100] + +-- 428 +BEGIN TRANSACTION; + CREATE TABLE t (c duration); + INSERT INTO t VALUES (duration("1s")), (duration("1m")), (duration("1h")); +COMMIT; +SELECT c, string(c) FROM t ORDER BY c; +|?c, s +[1s 1s] +[1m0s 1m0s] +[1h0m0s 1h0m0s] + +-- 429 +BEGIN TRANSACTION; + CREATE TABLE t (c time); + INSERT INTO t VALUES (date(2013, 11, 26, 10, 18, 5, 999999999, "CET")); +COMMIT; +SELECT since(c) > duration("24h") FROM t; +|b +[true] + +-- 430 +BEGIN TRANSACTION; + CREATE TABLE t (c time); + INSERT INTO t VALUES (date(2013, 11, 26, 10, 32, 5, 999999999, "CET")); +COMMIT; +SELECT !(since(c) < duration("24h")) FROM t; +|b +[true] + +-- 431 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("42h21m11.999999994s"), + duration("42h21m11.999999995s"), + duration("42h21m11.999999996s"), + ), + ; +COMMIT; +SELECT a > a, a > b, a > c, b > a, b > b, b > c, c > a, c > b, c > c FROM t; +|b, b, b, b, b, b, b, b, b +[false false false true false false true true false] + +-- 432 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("42h21m11.999999994s"), + duration("42h21m11.999999995s"), + duration("42h21m11.999999996s"), + ), + ; +COMMIT; +SELECT a < a, a < b, a < c, b < a, b < b, b < c, c < a, c < b, c < c FROM t; +|b, b, b, b, b, b, b, b, b +[false true true false false true false false false] + +-- 433 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("42h21m11.999999994s"), + duration("42h21m11.999999995s"), + duration("42h21m11.999999996s"), + ), + ; +COMMIT; +SELECT a <= a, a <= b, a <= c, b <= a, b <= b, b <= c, c <= a, c <= b, c <= c FROM t; +|b, b, b, b, b, b, b, b, b +[true true true false true true false false true] + +-- 434 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("42h21m11.999999994s"), + duration("42h21m11.999999995s"), + duration("42h21m11.999999996s"), + ), + ; +COMMIT; +SELECT a >= a, a >= b, a >= c, b >= a, b >= b, b >= c, c >= a, c >= b, c >= c FROM t; +|b, b, b, b, b, b, b, b, b +[true false false true true false true true true] + +-- 435 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("42h21m11.999999994s"), + duration("42h21m11.999999995s"), + duration("42h21m11.999999996s"), + ), + ; +COMMIT; +SELECT a != a, a != b, a != c, b != a, b != b, b != c, c != a, c != b, c != c FROM t; +|b, b, b, b, b, b, b, b, b +[false true true true false true true true false] + +-- 436 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("42h21m11.999999994s"), + duration("42h21m11.999999995s"), + duration("42h21m11.999999996s"), + ), + ; +COMMIT; +SELECT a == a, a == b, a == c, b == a, b == b, b == c, c == a, c == b, c == c FROM t; +|b, b, b, b, b, b, b, b, b +[true false false false true false false false true] + +-- 437 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("5h"), + duration("3m"), + duration("2s"), + ), + ; +COMMIT; +SELECT b+c, a+c, a+b, a+b+c FROM t; +|?, ?, ?, ? +[3m2s 5h0m2s 5h3m0s 5h3m2s] + +-- 438 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("5h"), + duration("3m"), + duration("2s"), + ), + ; +COMMIT; +SELECT b-c, a-c, a-b, a-b-c FROM t; +|?, ?, ?, ? +[2m58s 4h59m58s 4h57m0s 4h56m58s] + +-- 439 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("5h"), + duration("3m"), + duration("2s"), + ), + ; +COMMIT; +SELECT a>>1, b>>1, c>>1 FROM t; +|?, ?, ? +[2h30m0s 1m30s 1s] + +-- 440 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("5h"), + duration("3m"), + duration("2s"), + ), + ; +COMMIT; +SELECT a<<1, b<<1, c<<1 FROM t; +|?, ?, ? +[10h0m0s 6m0s 4s] + +-- 441 +BEGIN TRANSACTION; + CREATE TABLE t (a duration); + INSERT INTO t VALUES ( + duration("257ns"), + ), + ; +COMMIT; +SELECT a & 255 FROM t; +|? +[1ns] + +-- 442 +BEGIN TRANSACTION; + CREATE TABLE t (a duration); + INSERT INTO t VALUES ( + duration("1ns"), + ), + ; +COMMIT; +SELECT a ∨ 256 FROM t; +|? +[257ns] + +-- 443 +BEGIN TRANSACTION; + CREATE TABLE t (a duration); + INSERT INTO t VALUES ( + duration(0x731), + ), + ; +COMMIT; +SELECT a &^ 0xd30 FROM t; +|? +[513ns] + +-- 444 +BEGIN TRANSACTION; + CREATE TABLE t (a duration); + INSERT INTO t VALUES ( + duration("3h2m1s"), + ), + ; +COMMIT; +SELECT a % duration("2h"), a % duration("1m") FROM t; +|?, ? +[1h2m1s 1s] + +-- 445 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("5h"), + duration("3m"), + duration("2s"), + ), + ; +COMMIT; +SELECT a/2, b/2, c/2 FROM t; +|?, ?, ? +[2h30m0s 1m30s 1s] + +-- 446 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("5h"), + duration("3m"), + duration("2s"), + ), + ; +COMMIT; +SELECT a*2, 2*b, c*2 FROM t; +|?, ?, ? +[10h0m0s 6m0s 4s] + +-- 447 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("1ns"), + duration("3ns"), + duration("5ns"), + ), + ; +COMMIT; +SELECT ^a, ^b, ^c FROM t; +|?, ?, ? +[-2ns -4ns -6ns] + +-- 448 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("1ns"), + duration("3ns"), + duration("5ns"), + ), + ; +COMMIT; +SELECT +a, +b, +c FROM t; +|?, ?, ? +[1ns 3ns 5ns] + +-- 449 +BEGIN TRANSACTION; + CREATE TABLE t (a duration, b duration, c duration); + INSERT INTO t VALUES ( + duration("1ns"), + duration("3ns"), + duration("5ns"), + ), + ; +COMMIT; +SELECT -a, -b, -c FROM t; +|?, ?, ? +[-1ns -3ns -5ns] + +-- 450 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b time, c time); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999994, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999995, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999996, "CET"), + ), + ; +COMMIT; +SELECT a > a, a > b, a > c, b > a, b > b, b > c, c > a, c > b, c > c FROM t; +|b, b, b, b, b, b, b, b, b +[false false false true false false true true false] + +-- 451 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b time, c time); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999994, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999995, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999996, "CET"), + ), + ; +COMMIT; +SELECT a < a, a < b, a < c, b < a, b < b, b < c, c < a, c < b, c < c FROM t; +|b, b, b, b, b, b, b, b, b +[false true true false false true false false false] + +-- 452 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b time, c time); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999994, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999995, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999996, "CET"), + ), + ; +COMMIT; +SELECT a <= a, a <= b, a <= c, b <= a, b <= b, b <= c, c <= a, c <= b, c <= c FROM t; +|b, b, b, b, b, b, b, b, b +[true true true false true true false false true] + +-- 453 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b time, c time); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999994, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999995, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999996, "CET"), + ), + ; +COMMIT; +SELECT a >= a, a >= b, a >= c, b >= a, b >= b, b >= c, c >= a, c >= b, c >= c FROM t; +|b, b, b, b, b, b, b, b, b +[true false false true true false true true true] + +-- 454 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b time, c time); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999994, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999995, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999996, "CET"), + ), + ; +COMMIT; +SELECT a != a, a != b, a != c, b != a, b != b, b != c, c != a, c != b, c != c FROM t; +|b, b, b, b, b, b, b, b, b +[false true true true false true true true false] + +-- 455 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b time, c time); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999994, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999995, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999996, "CET"), + ), + ; +COMMIT; +SELECT a == a, a == b, a == c, b == a, b == b, b == c, c == a, c == b, c == c FROM t; +|b, b, b, b, b, b, b, b, b +[true false false false true false false false true] + +-- 456 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b duration); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999999, "CET"), + duration("3h2m1s"), + ), + ; +COMMIT; +SELECT formatTime(a+b, "2006-01-02 15:04:05.999999999 -0700") FROM t; +|s +[2013-11-27 15:03:03.999999999 +0100] + +-- 457 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b duration); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999999, "CET"), + duration("3h2m1s"), + ), + ; +COMMIT; +SELECT formatTime(b+a, "2006-01-02 15:04:05.999999999 -0700") FROM t; +|s +[2013-11-27 15:03:03.999999999 +0100] + +-- 458 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b duration); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999999, "CET"), + duration("3h2m1s"), + ), + ; +COMMIT; +SELECT a+a FROM t; +||invalid operation + +-- 459 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b time); + INSERT INTO t VALUES ( + date(2013, 11, 27, 13, 2, 3, 999999999, "CET"), + date(2013, 11, 27, 12, 1, 2, 999999999, "CET"), + ), + ; +COMMIT; +SELECT a-b FROM t; +|? +[1h1m1s] + +-- 460 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b duration); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999999, "CET"), + duration("3h2m1s"), + ), + ; +COMMIT; +SELECT formatTime(a-b, "2006-01-02 15:04:05.999999999 -0700") FROM t; +|s +[2013-11-27 08:59:01.999999999 +0100] + +-- 461 +BEGIN TRANSACTION; + CREATE TABLE t (a time, b duration); + INSERT INTO t VALUES ( + date(2013, 11, 27, 12, 1, 2, 999999999, "CET"), + duration("3h2m1s"), + ), + ; +COMMIT; +SELECT b-a FROM t; +||invalid operation + +-- 462 +BEGIN TRANSACTION; + CREATE TABLE t (a duration); + INSERT INTO t VALUES ( + duration("3h2m1.5s"), + ), + ; +COMMIT; +SELECT hours(a), minutes(a), seconds(a), nanoseconds(a) FROM t; +|g, g, g, l +[3.03375 182.025 10921.5 10921500000000] + +-- 463 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES (now()-duration("1s")); +COMMIT; +SELECT a < now(), now() > a, a >= now(), now() <= a FROM t; +|b, b, b, b +[true true false false] + +-- 464 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04pm (MST)", "Nov 27, 2013 at 2:07pm (CET)")), + (parseTime("2006-Jan-02", "2013-Nov-27")), + ; +COMMIT; +SELECT formatTime(a, "2006-01-02 15:04:05.999999999 -0700") as a FROM t ORDER BY a; +|sa +[2013-11-27 00:00:00 +0000] +[2013-11-27 14:07:00 +0100] + +-- 465 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04pm (MST)", "Nov 27, 2013 at 2:07pm (CET)")), + (parseTime("2006-Jan-02", "2013-Nov-27")), + ; +COMMIT; +SELECT hour(a) AS y FROM t ORDER BY y; +|ly +[0] +[14] + +-- 466 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04pm (MST)", "Nov 27, 2013 at 2:07pm (CET)")), + (parseTime("2006-Jan-02", "2013-Nov-27")), + ; +COMMIT; +SELECT minute(a) AS y FROM t ORDER BY y; +|ly +[0] +[7] + +-- 467 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04:05pm (MST)", "Nov 27, 2013 at 2:07:31pm (CET)")), + (parseTime("2006-Jan-02", "2013-Nov-27")), + ; +COMMIT; +SELECT second(a) AS y FROM t ORDER BY y; +|ly +[0] +[31] + +-- 468 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04:05pm (MST)", "Nov 27, 2013 at 2:07:31.123456789pm (CET)")), + (parseTime("2006-Jan-02", "2013-Nov-27")), + ; +COMMIT; +SELECT nanosecond(a) AS y FROM t ORDER BY y; +|ly +[0] +[123456789] + +-- 469 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04:05pm (MST)", "Nov 27, 2013 at 2:07:31.123456789pm (CET)")), + (parseTime("2006-Jan-02", "2014-Nov-28")), + ; +COMMIT; +SELECT year(a) AS y FROM t ORDER BY y; +|ly +[2013] +[2014] + +-- 470 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04:05pm (MST)", "Nov 27, 2013 at 2:07:31.123456789pm (CET)")), + (parseTime("2006-Jan-02", "2014-Nov-28")), + ; +COMMIT; +SELECT day(a) AS y FROM t ORDER BY y; +|ly +[27] +[28] + +-- 471 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04:05pm (MST)", "Nov 27, 2013 at 2:07:31.123456789pm (CET)")), + (parseTime("2006-Jan-02", "2014-Dec-28")), + ; +COMMIT; +SELECT month(a) AS y FROM t ORDER BY y; +|ly +[11] +[12] + +-- 472 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04:05pm (MST)", "Nov 27, 2013 at 2:07:31.123456789pm (CET)")), + (parseTime("2006-Jan-02", "2013-Sep-08")), + ; +COMMIT; +SELECT weekday(a) AS y FROM t ORDER BY y; +|ly +[0] +[3] + +-- 473 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04:05pm (MST)", "Feb 1, 2013 at 2:07:31.123456789pm (CET)")), + (parseTime("2006-Jan-02", "2014-Feb-02")), + ; +COMMIT; +SELECT yearDay(a) AS y FROM t ORDER BY y; +|ly +[32] +[33] + +-- 474 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04pm (MST)", "Feb 1, 2013 at 2:07pm (CET)")), + (parseTime("2006-Jan-02", "2014-Feb-02")), + ; +COMMIT; +SELECT timeIn(a, ""), timeIn(a, "UTC") AS y FROM t ORDER BY y; +|?, ?y +[2013-02-01 13:07:00 +0000 UTC 2013-02-01 13:07:00 +0000 UTC] +[2014-02-02 00:00:00 +0000 UTC 2014-02-02 00:00:00 +0000 UTC] + +-- 475 +BEGIN TRANSACTION; + CREATE TABLE t (a time); + INSERT INTO t VALUES + (parseTime("Jan 2, 2006 at 3:04pm (MST)", "Feb 1, 2013 at 2:07pm (CET)")), + (parseTime("2006-Jan-02", "2014-Feb-02")), + ; +COMMIT; +SELECT formatTime(timeIn(a, "UTC"), "Jan 2, 2006 at 3:04pm (UTC)") AS y FROM t ORDER BY y; +|sy +[Feb 1, 2013 at 1:07pm (UTC)] +[Feb 2, 2014 at 12:00am (UTC)] + +-- 476 +BEGIN TRANSACTION; + BEGIN TRANSACTION; + COMMIT; +COMMIT; +SELECT * FROM t; +||does not exist + +-- 477 +BEGIN TRANSACTION; + BEGIN TRANSACTION; + ROLLBACK; +COMMIT; +SELECT * FROM t; +||does not exist + +-- 478 // https://github.com/cznic/ql/issues/23 +BEGIN TRANSACTION; + CREATE TABLE t (a int, b string, t time); + INSERT INTO t VALUES (1, "a", parseTime("Jan 2, 2006 at 3:04pm (MST)", "Jan 12, 2014 at 6:26pm (CET)")); + INSERT INTO t VALUES (2, "b", parseTime("Jan 2, 2006 at 3:04pm (MST)", "Jan 12, 2014 at 6:27pm (CET)")); + UPDATE t b = "hello" WHERE a == 1; +COMMIT; +SELECT a, b, formatTime(t, "2006-01-02 15:04:05.999999999 -0700") as t FROM t; +|la, sb, st +[2 b 2014-01-12 18:27:00 +0100] +[1 hello 2014-01-12 18:26:00 +0100] + +-- 479 // https://github.com/cznic/ql/issues/23 +BEGIN TRANSACTION; + CREATE TABLE t (a int, b string, t time); + INSERT INTO t VALUES (1, "a", parseTime("Jan 2, 2006 at 3:04pm (MST)", "Jan 12, 2014 at 6:26pm (CET)")); + INSERT INTO t VALUES (2, "b", parseTime("Jan 2, 2006 at 3:04pm (MST)", "Jan 12, 2014 at 6:27pm (CET)")); + UPDATE t + b = "hello", + t = parseTime("Jan 2, 2006 at 3:04pm (MST)", "Jan 12, 2014 at 6:28pm (CET)"), + WHERE a == 1; +COMMIT; +SELECT a, b, formatTime(t, "2006-01-02 15:04:05.999999999 -0700") as t FROM t; +|la, sb, st +[2 b 2014-01-12 18:27:00 +0100] +[1 hello 2014-01-12 18:28:00 +0100] + +-- 480 // https://github.com/cznic/ql/issues/23 +BEGIN TRANSACTION; + CREATE TABLE t (a int, b string, d duration); + INSERT INTO t VALUES (1, "a", duration("1m")); + INSERT INTO t VALUES (2, "b", duration("2m")); + UPDATE t b = "hello" WHERE a == 1; +COMMIT; +SELECT * FROM t; +|la, sb, ?d +[2 b 2m0s] +[1 hello 1m0s] + +-- 481 // https://github.com/cznic/ql/issues/23 +BEGIN TRANSACTION; + CREATE TABLE t (a int, b string, d duration); + INSERT INTO t VALUES (1, "a", duration("1m")); + INSERT INTO t VALUES (2, "b", duration("2m")); + UPDATE t + b = "hello", + d = duration("3m"), + WHERE a == 1; +COMMIT; +SELECT * FROM t; +|la, sb, ?d +[2 b 2m0s] +[1 hello 3m0s] + +-- 482 // https://github.com/cznic/ql/issues/24 +BEGIN TRANSACTION; + CREATE TABLE t (c complex128); + INSERT INTO t VALUES + (2+complex128(1)), + (22+complex(0, 1)), + ; +COMMIT; +SELECT * FROM t ORDER BY real(c); +|dc +[(3+0i)] +[(22+1i)] + +-- 483 +BEGIN TRANSACTION; + CREATE TABLE t (s string, substr string); + INSERT INTO t VALUES + ("seafood", "foo"), + ("seafood", "bar"), + ("seafood", ""), + ("", ""), + ; +COMMIT; +SELECT id() as i, contains(42, substr) FROM t ORDER BY i; +||invalid .* 42 + +-- 484 +BEGIN TRANSACTION; + CREATE TABLE t (s string, substr string); + INSERT INTO t VALUES + ("seafood", "foo"), + ("seafood", "bar"), + ("seafood", ""), + ("", ""), + ; +COMMIT; +SELECT id() as i, contains(s, true) FROM t ORDER BY i; +||invalid .* true + +-- 485 +BEGIN TRANSACTION; + CREATE TABLE t (s string, substr string); + INSERT INTO t VALUES + ("seafood", "foo"), + ("seafood", "bar"), + ("seafood", ""), + ("", ""), + ("", NULL), + ("foo", NULL), + (NULL, ""), + (NULL, "foo"), + (NULL, NULL), + ; +COMMIT; +SELECT id() as i, contains(s, substr) FROM t ORDER BY i; +|li, b +[1 true] +[2 false] +[3 true] +[4 true] +[5 ] +[6 ] +[7 ] +[8 ] +[9 ] + +-- 486 +BEGIN TRANSACTION; + CREATE TABLE t (s string, prefix string); + INSERT INTO t VALUES + ("", ""), + ("f", ""), + ("", "foo"), + ("f", "foo"), + ("fo", "foo"), + ("foo", "foo"), + ("fooo", "foo"), + ; +COMMIT; +SELECT id() as i, hasPrefix(42, prefix) FROM t ORDER BY i; +||invalid .* 42 + +-- 487 +BEGIN TRANSACTION; + CREATE TABLE t (s string, prefix string); + INSERT INTO t VALUES + ("", ""), + ("f", ""), + ("", "foo"), + ("f", "foo"), + ("fo", "foo"), + ("foo", "foo"), + ("fooo", "foo"), + ; +COMMIT; +SELECT id() as i, hasPrefix(s, false) FROM t ORDER BY i; +||invalid .* false + +-- 488 +BEGIN TRANSACTION; + CREATE TABLE t (s string, prefix string); + INSERT INTO t VALUES + ("", ""), + ("f", ""), + ("", "foo"), + ("f", "foo"), + ("fo", "foo"), + ("foo", "foo"), + ("fooo", "foo"), + ("", NULL), + ("foo", NULL), + (NULL, ""), + (NULL, "foo"), + (NULL, NULL), + ; +COMMIT; +SELECT id() as i, hasPrefix(s, prefix) FROM t ORDER BY i; +|li, b +[1 true] +[2 true] +[3 false] +[4 false] +[5 false] +[6 true] +[7 true] +[8 ] +[9 ] +[10 ] +[11 ] +[12 ] + +-- 489 +BEGIN TRANSACTION; + CREATE TABLE t (s string, suffix string); + INSERT INTO t VALUES + ("", ""), + ("f", ""), + ("x", "foo"), + ("xf", "foo"), + ("xfo", "foo"), + ("xfoo", "foo"), + ("xfooo", "foo"), + ; +COMMIT; +SELECT id() as i, hasSuffix(42, suffix) FROM t ORDER BY i; +||invalid .* 42 + +-- 490 +BEGIN TRANSACTION; + CREATE TABLE t (s string, suffix string); + INSERT INTO t VALUES + ("", ""), + ("f", ""), + ("x", "foo"), + ("xf", "foo"), + ("xfo", "foo"), + ("xfoo", "foo"), + ("xfooo", "foo"), + ; +COMMIT; +SELECT id() as i, hasSuffix(s, true) FROM t ORDER BY i; +||invalid .* true + +-- 491 +BEGIN TRANSACTION; + CREATE TABLE t (s string, suffix string); + INSERT INTO t VALUES + ("", ""), + ("f", ""), + ("x", "foo"), + ("xf", "foo"), + ("xfo", "foo"), + ("xfoo", "foo"), + ("xfooo", "foo"), + ("", NULL), + ("foo", NULL), + (NULL, ""), + (NULL, "foo"), + (NULL, NULL), + ; +COMMIT; +SELECT id() as i, hasSuffix(s, suffix) FROM t ORDER BY i; +|li, b +[1 true] +[2 true] +[3 false] +[4 false] +[5 false] +[6 true] +[7 false] +[8 ] +[9 ] +[10 ] +[11 ] +[12 ] + +-- 492 // issue #27 +BEGIN TRANSACTION; + DROP TABLE nonexistent; +COMMIT; +||does not exist + +-- 493 // issue #27 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + DROP TABLE IF EXISTS nonexistent; +COMMIT; +SELECT * FROM t; +|?i + +-- 494 // issue #27 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE TABLE t (i int); +COMMIT; +||exist + +-- 495 // issue #27 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE TABLE IF NOT EXISTS t (s string); +COMMIT; +SELECT * FROM t; +|?i + +-- 496 // issue #28 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); + ALTER TABLE t ADD s string; +COMMIT; +SELECT * FROM t; +|li, ?s +[42 ] + +-- 497 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +BEGIN TRANSACTION; + INSERT INTO t VALUES(1000); + BEGIN TRANSACTION; + INSERT INTO t VALUES(2000); + COMMIT; + INSERT INTO t VALUES(3000); +COMMIT; +SELECT * FROM t; +|li +[3000] +[2000] +[1000] + +-- 498 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +BEGIN TRANSACTION; + INSERT INTO t VALUES(1000); + BEGIN TRANSACTION; + INSERT INTO t VALUES(2000); + ROLLBACK; + INSERT INTO t VALUES(3000); +COMMIT; +SELECT * FROM t; +|li +[3000] +[1000] + +-- 499 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES(42, "foo"); + ALTER TABLE t DROP COLUMN i; +COMMIT; +SELECT * FROM t; +|ss +[foo] + +-- 500 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES(42, "foo"); + ALTER TABLE t DROP COLUMN s; +COMMIT; +SELECT * FROM t; +|li +[42] + +-- 501 // new spec rule: table must have at least 1 column +BEGIN TRANSACTION; + CREATE TABLE t (c int); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t DROP COLUMN c; +COMMIT; +SELECT * FROM t; +||cannot drop.*column + +-- 502 // fixed bug +BEGIN TRANSACTION; + CREATE TABLE t (c int, s string); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t DROP COLUMN s; +ROLLBACK; +SELECT * FROM t; +|?c, ?s + +-- 503 // fixed bug +BEGIN TRANSACTION; + CREATE TABLE t (c int, s string); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t ADD b bool; +ROLLBACK; +SELECT * FROM t; +|?c, ?s + +-- 504 // fixed bug +BEGIN TRANSACTION; + CREATE TABLE t (c int, s string); +COMMIT; +BEGIN TRANSACTION; + DROP TABLE t; +ROLLBACK; +SELECT * FROM t; +|?c, ?s + +-- 505 // fixed bug +BEGIN TRANSACTION; + CREATE INDEX x ON t (qty()); +COMMIT; +||only .* id + +-- 506 +BEGIN TRANSACTION; + CREATE INDEX x ON t (qty); +COMMIT; +||table.*not exist + +-- 507 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX x ON t (qty); +COMMIT; +||column.*not exist + +-- 508 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX x ON t (id()); +COMMIT; +SELECT * FROM t; +|?c + +-- 509 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX y ON t (c); +COMMIT; +SELECT * FROM t; +|?c + +-- 510 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX x ON t (id()); + CREATE INDEX y ON t (id()); +COMMIT; +SELECT * FROM t; +||already + +-- 511 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX x ON t (id()); + CREATE INDEX x ON t (c); +COMMIT; +SELECT * FROM t; +||already + +-- 512 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX x ON t (id()); + CREATE INDEX y ON t (c); +COMMIT; +SELECT * FROM t; +|?c + +-- 513 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX y ON t (c); + CREATE INDEX x ON t (id()); +COMMIT; +SELECT * FROM t; +|?c + +-- 514 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES(42); +COMMIT; +SELECT * FROM t; +|lc +[42] + +-- 515 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + INSERT INTO t VALUES(42); + CREATE INDEX x ON t (id()); +COMMIT; +SELECT * FROM t; +|lc +[42] + +-- 516 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + INSERT INTO t VALUES(42); + INSERT INTO t VALUES(24); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES(1); + CREATE INDEX i ON t (c); + INSERT INTO t VALUES(999); +COMMIT; +SELECT * FROM t ORDER BY id(); +|lc +[42] +[24] +[1] +[999] + +-- 517 +BEGIN TRANSACTION; + CREATE TABLE t (c int); + INSERT INTO t VALUES(42); + INSERT INTO t VALUES(24); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES(1); + CREATE INDEX i ON t (c); + INSERT INTO t VALUES(999); +COMMIT; +SELECT * FROM t ORDER BY c; +|lc +[1] +[24] +[42] +[999] + +-- 518 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX xid ON t (id()); + INSERT INTO t VALUES(42); + INSERT INTO t VALUES(24); + CREATE INDEX ii ON t (i); + INSERT INTO t VALUES(1); + INSERT INTO t VALUES(999); + UPDATE t i = 240 WHERE i == 24; + DELETE FROM t WHERE i == 240; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[42] +[999] + +-- 519 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX i ON t (i); +COMMIT; +||collision: i + +-- 520 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX t ON t (i); +COMMIT; +||collision: t + +-- 521 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE TABLE u (s string); + CREATE INDEX u ON t (i); +COMMIT; +||collision.*: u + +-- 522 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE TABLE u (s string); + CREATE INDEX z ON t (i); + CREATE INDEX z ON u (s); +COMMIT; +||already + +-- 523 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX u ON u (s); +COMMIT; +||collision: u + +-- 524 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX v ON u (v); +COMMIT; +||collision: v + +-- 525 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX s ON t (i); +COMMIT; +||collision.*: s + +-- 526 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX id ON t (i); +COMMIT; +SELECT * FROM t; +|?i + +-- 527 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + CREATE TABLE x (s string); +COMMIT; +||table t.*index x + +-- 528 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +BEGIN TRANSACTION; + INSERT INTO t VALUES(1000); + BEGIN TRANSACTION; + INSERT INTO t VALUES(2000); + ROLLBACK; + INSERT INTO t VALUES(3000); +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1000] +[3000] + +-- 529 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + TRUNCATE TABLE t; +COMMIT; +SELECT * FROM t; +|?i + +-- 530 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + DELETE FROM t; +COMMIT; +SELECT * FROM t; +|?i + +-- 531 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX xi ON t (i); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN i; + INSERT INTO t VALUES ("bar"); +COMMIT; +SELECT * FROM t ORDER BY s; +|ss +[bar] +[foo] + +-- 532 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX xs ON t (s); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN i; + INSERT INTO t VALUES ("bar"); +COMMIT; +SELECT * FROM t ORDER BY s; +|ss +[bar] +[foo] + +-- 533 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX xi ON t (i); + CREATE INDEX xs ON t (s); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN i; + INSERT INTO t VALUES ("bar"); +COMMIT; +SELECT * FROM t ORDER BY s; +|ss +[bar] +[foo] + +-- 534 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX xi ON t (i); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN s; + INSERT INTO t VALUES (24); +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[24] +[42] + +-- 535 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX xs ON t (s); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN s; + INSERT INTO t VALUES (24); +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[24] +[42] + +-- 536 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX xi ON t (i); + CREATE INDEX xs ON t (s); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN s; + INSERT INTO t VALUES (24); +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[24] +[42] + +-- 537 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN i; +COMMIT; +SELECT * FROM t; +|ss +[foo] + +-- 538 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES (42, "foo"); + ALTER TABLE t DROP COLUMN s; +COMMIT; +SELECT * FROM t; +|li +[42] + +-- 539 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES (42, "foo"); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t DROP COLUMN i; +COMMIT; +SELECT * FROM t; +|ss +[foo] + +-- 540 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES (42, "foo"); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t DROP COLUMN s; +COMMIT; +SELECT * FROM t; +|li +[42] + +-- 541 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42, "foo"); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t DROP COLUMN s; +COMMIT; +BEGIN TRANSACTION; + INSERT INTO t VALUES (24); +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[24] +[42] + +-- 542 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42, "foo"); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t DROP COLUMN i; +COMMIT; +BEGIN TRANSACTION; + INSERT INTO t VALUES ("bar"); +COMMIT; +SELECT * FROM t ORDER BY s; +|ss +[bar] +[foo] + +-- 543 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + INSERT INTO t SELECT 10*i FROM t; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[42] +[420] + +-- 544 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + INSERT INTO t SELECT 10*i FROM t; + DROP INDEX none; +COMMIT; +||index none does not exist + +-- 545 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + INSERT INTO t SELECT 10*i FROM t; + DROP INDEX x; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[42] +[420] + +-- 546 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + INSERT INTO t SELECT 10*i FROM t; +COMMIT; +BEGIN TRANSACTION; + DROP INDEX x; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[42] +[420] + +-- 547 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + ALTER TABLE t ADD s string; +COMMIT; +SELECT * FROM t; +|?i, ?s + +-- 548 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + ALTER TABLE t ADD s string; +COMMIT; +SELECT * FROM t ORDER BY i; +|?i, ?s + +-- 549 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t ADD s string; +COMMIT; +SELECT * FROM t ORDER BY s; +|?i, ?s + +-- 550 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); +COMMIT; +SELECT * FROM x; +|?x +[42] + +-- 551 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (42); + INSERT INTO t VALUES (420); +COMMIT; +SELECT * FROM x; +|?x +[42] +[420] + +-- 552 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (420); + INSERT INTO t VALUES (42); +COMMIT; +SELECT * FROM x; +|?x +[42] +[420] + +-- 553 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (420); + INSERT INTO t VALUES (42); + INSERT INTO t VALUES (100); +COMMIT; +SELECT * FROM x; +|?x +[42] +[100] +[420] + +-- 554 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (420); + INSERT INTO t VALUES (42); + INSERT INTO t VALUES (100); + DELETE FROM t WHERE i == 100; +COMMIT; +SELECT * FROM x; +|?x +[42] +[420] + +-- 555 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (420); + INSERT INTO t VALUES (42); + INSERT INTO t VALUES (100); +COMMIT; +BEGIN TRANSACTION; + DELETE FROM t WHERE i == 100; +COMMIT; +SELECT * FROM x; +|?x +[42] +[420] + +-- 556 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (1); + INSERT INTO t VALUES (2); + INSERT INTO t VALUES (3); +COMMIT; +SELECT * FROM x; +|?x +[1] +[2] +[3] + +-- 557 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (1); + INSERT INTO t VALUES (3); + INSERT INTO t VALUES (2); +COMMIT; +SELECT * FROM x; +|?x +[1] +[2] +[3] + +-- 558 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (2); + INSERT INTO t VALUES (1); + INSERT INTO t VALUES (3); +COMMIT; +SELECT * FROM x; +|?x +[1] +[2] +[3] + +-- 559 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (2); + INSERT INTO t VALUES (3); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * FROM x; +|?x +[1] +[2] +[3] + +-- 560 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (3); + INSERT INTO t VALUES (1); + INSERT INTO t VALUES (2); +COMMIT; +SELECT * FROM x; +|?x +[1] +[2] +[3] + +-- 561 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (3); + INSERT INTO t VALUES (2); + INSERT INTO t VALUES (1); +COMMIT; +SELECT * FROM x; +|?x +[1] +[2] +[3] + +-- 562 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (bigint(1) << 100); + INSERT INTO t VALUES (bigint(1) << 100 - 1); + INSERT INTO t VALUES (bigint(1) << (256*8)); + INSERT INTO t VALUES (bigint(1) << 100 + 1); + INSERT INTO t VALUES (bigint(1) << 10); +COMMIT; +SELECT * FROM x; +|?x +[1024] +[1267650600228229401496703205375] +[1267650600228229401496703205376] +[1267650600228229401496703205377] +[32317006071311007300714876688669951960444102669715484032130345427524655138867890893197201411522913463688717960921898019494119559150490921095088152386448283120630877367300996091750197750389652106796057638384067568276792218642619756161838094338476170470581645852036305042887575891541065808607552399123930385521914333389668342420684974786564569494856176035326322058077805659331026192708460314150258592864177116725943603718461857357598351152301645904403697613233287231227125684710820209725157101726931323469678542580656697935045997268352998638215525166389437335543602135433229604645318478604952148193555853611059596230656] + +-- 563 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (bigint(1) << 100); + INSERT INTO t VALUES (bigint(1) << 100 - 1); + INSERT INTO t VALUES (bigint(1) << (256*8)); + INSERT INTO t VALUES (bigint(1) << 100 + 1); + INSERT INTO t VALUES (bigint(1) << 10); +COMMIT; +BEGIN TRANSACTION; + DELETE FROM t WHERE i == (bigint(1) << (256*8)); +COMMIT; +SELECT * FROM x; +|?x +[1024] +[1267650600228229401496703205375] +[1267650600228229401496703205376] +[1267650600228229401496703205377] + +-- 564 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (bigint(1) << 100); + INSERT INTO t VALUES (bigint(1) << 100 - 1); + INSERT INTO t VALUES (bigint(1) << 100 + 1); + INSERT INTO t VALUES (bigint(1) << 10); +COMMIT; +BEGIN TRANSACTION; + UPDATE t + i = i+10, + WHERE i == bigint(1) << 100; +COMMIT; +SELECT * FROM x; +|?x +[1024] +[1267650600228229401496703205375] +[1267650600228229401496703205377] +[1267650600228229401496703205386] + +-- 565 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (bigint(1) << 100); + INSERT INTO t VALUES (bigint(1) << 100 - 1); + INSERT INTO t VALUES (bigint(1) << (256*8)); + INSERT INTO t VALUES (bigint(1) << 100 + 1); + INSERT INTO t VALUES (bigint(1) << 10); +COMMIT; +BEGIN TRANSACTION; + UPDATE t + i = 42, + WHERE i == bigint(1) << (256*8); +COMMIT; +SELECT * FROM x; +|?x +[42] +[1024] +[1267650600228229401496703205375] +[1267650600228229401496703205376] +[1267650600228229401496703205377] + +-- 566 +BEGIN TRANSACTION; + CREATE TABLE t (i bigint); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES (bigint(1) << 100); + INSERT INTO t VALUES (bigint(1) << 100 - 1); + INSERT INTO t VALUES (bigint(42)); + INSERT INTO t VALUES (bigint(1) << 100 + 1); + INSERT INTO t VALUES (bigint(1) << 10); +COMMIT; +BEGIN TRANSACTION; + UPDATE t + i = bigint(1) << (256*8), + WHERE i == 42; +COMMIT; +SELECT * FROM x; +|?x +[1024] +[1267650600228229401496703205375] +[1267650600228229401496703205376] +[1267650600228229401496703205377] +[32317006071311007300714876688669951960444102669715484032130345427524655138867890893197201411522913463688717960921898019494119559150490921095088152386448283120630877367300996091750197750389652106796057638384067568276792218642619756161838094338476170470581645852036305042887575891541065808607552399123930385521914333389668342420684974786564569494856176035326322058077805659331026192708460314150258592864177116725943603718461857357598351152301645904403697613233287231227125684710820209725157101726931323469678542580656697935045997268352998638215525166389437335543602135433229604645318478604952148193555853611059596230656] + +-- 567 +BEGIN TRANSACTION; + CREATE TABLE t (b blob); + CREATE INDEX x ON t (b); + INSERT INTO t VALUES (blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" // > shortBlob + )); + DROP TABLE t; +COMMIT; +SELECT * FROM t; +||does not exist + +-- 568 +BEGIN TRANSACTION; + CREATE TABLE t (b blob); + CREATE INDEX x ON t (b); + INSERT INTO t VALUES (blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" // > shortBlob + )); + DROP TABLE t; +COMMIT; +BEGIN TRANSACTION; + DROP TABLE t; +COMMIT; +SELECT * FROM t; +||does not exist + +-- 569 +BEGIN TRANSACTION; + CREATE TABLE t (b blob); + CREATE INDEX x ON t (b); + INSERT INTO t VALUES (blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" // > shortBlob + )); + DROP INDEX x; +COMMIT; +SELECT len(string(b)) AS n FROM t; +|ln +[320] + +-- 570 +BEGIN TRANSACTION; + CREATE TABLE t (b blob); + CREATE INDEX x ON t (b); + INSERT INTO t VALUES (blob( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" // > shortBlob + )); +COMMIT; +BEGIN TRANSACTION; + DROP INDEX x; +COMMIT; +SELECT len(string(b)) AS n FROM t; +|ln +[320] + +-- 571 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); + ALTER TABLE t ADD s string; +COMMIT; +SELECT * FROM t; +|li, ?s +[42 ] + +-- 572 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); +COMMIT; +BEGIN TRANSACTION; + ALTER TABLE t ADD s string; +COMMIT; +SELECT * FROM t; +|li, ?s +[42 ] + +-- 573 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +SELECT * FROM q.t; +||syntax error + +-- 574 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); +COMMIT; +SELECT * FROM t AS u; +|li +[42] + +-- 575 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); +COMMIT; +SELECT u.x FROM t AS u; +||unknown field u.x + +-- 576 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); +COMMIT; +SELECT u.i FROM t AS u; +||unknown field u.i + +-- 577 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); +COMMIT; +SELECT i FROM t AS u; +|li +[42] + +-- 578 +BEGIN TRANSACTION; + CREATE TABLE t (i int, b bool); + CREATE INDEX x ON t (b); + INSERT INTO t VALUES(24, false); + INSERT INTO t VALUES(333, NULL); + INSERT INTO t VALUES(42, true); + INSERT INTO t VALUES(240, false); + INSERT INTO t VALUES(420, true); +COMMIT; +SELECT i FROM t WHERE b ORDER BY i; +|li +[42] +[420] + +-- 579 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(10, "foo"); +COMMIT; +SELECT * FROM t WHERE i < "30"; +||cannot.*compar + +-- 580 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM x; +|?x +[] +[] +[10] +[10] +[20] +[20] +[30] +[30] +[40] +[40] +[50] +[50] + +-- 581 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE i < 30; +|li +[10] +[20] +[20] +[10] + +-- 582 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE i < 30; +|li +[10] +[10] +[20] +[20] + +-- 583 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i <= 30; +|li +[10] +[20] +[30] +[30] +[20] +[10] + +-- 584 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i <= 30; +|li +[10] +[10] +[20] +[20] +[30] +[30] + +-- 585 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE i == 30; +|li +[30] +[30] + +-- 586 // index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE i == 30; +|li +[30] +[30] + +-- 587 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i >= 30; +|li +[50] +[40] +[30] +[30] +[40] +[50] + +-- 588 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i >= 30; +|li +[30] +[30] +[40] +[40] +[50] +[50] + +-- 589 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i > 30; +|li +[50] +[40] +[40] +[50] + +-- 590 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i > 30; +|li +[50] +[50] +[40] +[40] + +-- 591 +BEGIN TRANSACTION; + CREATE TABLE t (i int, b bool); + INSERT INTO t VALUES(24, false); + INSERT INTO t VALUES(333, NULL); + INSERT INTO t VALUES(42, true); + INSERT INTO t VALUES(240, false); + INSERT INTO t VALUES(420, true); +COMMIT; +SELECT i FROM t WHERE !b ORDER BY i; +|li +[24] +[240] + +-- 592 +BEGIN TRANSACTION; + CREATE TABLE t (i int, b bool); + CREATE INDEX x ON t (b); + INSERT INTO t VALUES(24, false); + INSERT INTO t VALUES(333, NULL); + INSERT INTO t VALUES(42, true); + INSERT INTO t VALUES(240, false); + INSERT INTO t VALUES(420, true); +COMMIT; +SELECT i FROM t WHERE !b ORDER BY i; +|li +[24] +[240] + +-- 593 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE i < $1; // 30 +|li +[10] +[10] +[20] +[20] + +-- 594 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i <= $1; // 30 +|li +[10] +[10] +[20] +[20] +[30] +[30] + +-- 595 // index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE i == $1; // 30 +|li +[30] +[30] + +-- 596 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i >= $1; // 30 +|li +[30] +[30] +[40] +[40] +[50] +[50] + +-- 597 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i > $1; // 30 +|li +[50] +[50] +[40] +[40] + +-- 598 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE $1 > i; // 30 +|li +[10] +[10] +[20] +[20] + +-- 599 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE $1 >= i; // 30 +|li +[10] +[10] +[20] +[20] +[30] +[30] + +-- 600 // index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE $1 == i; // 30 +|li +[30] +[30] + +-- 601 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE $1 <= i; // 30 +|li +[30] +[30] +[40] +[40] +[50] +[50] + +-- 602 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE $1 < i; // 30 +|li +[50] +[50] +[40] +[40] + +-- 603 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE 30 > i; +|li +[10] +[10] +[20] +[20] + +-- 604 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE 30 >= i; +|li +[10] +[10] +[20] +[20] +[30] +[30] + +-- 605 // index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT i FROM t WHERE 30 == i; +|li +[30] +[30] + +-- 606 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE 30 <= i; +|li +[30] +[30] +[40] +[40] +[50] +[50] + +-- 607 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE 30 < i; +|li +[50] +[50] +[40] +[40] + +-- 608 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i < 30; +|li +[20] +[10] + +-- 609 // ordered -> index is used +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE UNIQUE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(20); + INSERT INTO t VALUES(30); + INSERT INTO t VALUES(40); + INSERT INTO t VALUES(50); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i < 30; +|li +[10] +[20] + +-- 610 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE UNIQUE INDEX x ON t (i); + INSERT INTO t VALUES(NULL); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(10); + INSERT INTO t VALUES(NULL); +COMMIT; +SELECT * FROM t WHERE i < 30; +||duplicate + +-- 611 // Issue #34 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42); +COMMIT; +SELECT * FROM t WHERE i == $0; +||parameter.*non zero + +-- 612 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE TABLE __Table (i int); +COMMIT; +SELECT * FROM t; +||system table + +-- 613 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE TABLE __Column (i int); +COMMIT; +SELECT * FROM t; +||system table + +-- 614 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE TABLE __Index (i int); +COMMIT; +SELECT * FROM t; +||system table + +-- 615 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + DROP TABLE __Table; +COMMIT; +SELECT * FROM t; +||system table + +-- 616 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + DROP TABLE __Column; +COMMIT; +SELECT * FROM t; +||system table + +-- 617 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + DROP TABLE __Index; +COMMIT; +SELECT * FROM t; +||system table + +-- 618 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX __Table ON t (i); +COMMIT; +SELECT * FROM t; +||system table + +-- 619 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX __Column ON t (i); +COMMIT; +SELECT * FROM t; +||system table + +-- 620 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX __Index ON t (i); +COMMIT; +SELECT * FROM t; +||system table + +-- 621 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON __Table (Name); +COMMIT; +SELECT * FROM t; +||system table + +-- 622 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON __Column (Name); +COMMIT; +SELECT * FROM t; +||system table + +-- 623 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON __Index (Name); +COMMIT; +SELECT * FROM t; +||system table + +-- 624 +SELECT * FROM __Table; +|?Name, ?Schema + +-- 625 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); +COMMIT; +SELECT * FROM __Table ORDER BY Name; +|sName, sSchema +[t CREATE TABLE t (i int64, s string);] + +-- 626 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE TABLE u (b bool, i bigint, t time, d duration); +COMMIT; +SELECT * FROM __Table ORDER BY Name; +|sName, sSchema +[t CREATE TABLE t (i int64, s string);] +[u CREATE TABLE u (b bool, i bigint, t time, d duration);] + +-- 627 +SELECT * FROM __Column; +|?TableName, ?Ordinal, ?Name, ?Type + +-- 628 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); +COMMIT; +SELECT * FROM __Column ORDER BY TableName, Name; +|sTableName, lOrdinal, sName, sType +[t 1 i int64] +[t 2 s string] + +-- 629 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE TABLE u (b bool, i bigint, t time, d duration); +COMMIT; +SELECT * FROM __Column ORDER BY TableName, Ordinal; +|sTableName, lOrdinal, sName, sType +[t 1 i int64] +[t 2 s string] +[u 1 b bool] +[u 2 i bigint] +[u 3 t time] +[u 4 d duration] + +-- 630 +SELECT * FROM __Index; +|?TableName, ?ColumnName, ?Name, ?IsUnique + +-- 631 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); +COMMIT; +SELECT * FROM __Index ORDER BY TableName, Name; +|?TableName, ?ColumnName, ?Name, ?IsUnique + +-- 632 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (i); +COMMIT; +SELECT * FROM __Index ORDER BY TableName, ColumnName, Name; +|sTableName, sColumnName, sName, bIsUnique +[t i x false] + +-- 633 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (i); + CREATE INDEX id ON t (id()); + CREATE TABLE u (b bool, i bigint, t time, d duration); +COMMIT; +SELECT * FROM __Index ORDER BY TableName, ColumnName, Name; +|sTableName, sColumnName, sName, bIsUnique +[t i x false] +[t id() id false] + +-- 634 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (i); + CREATE INDEX id ON t (id()); + CREATE TABLE u (b bool, i bigint, t time, d duration); + CREATE INDEX z ON u (t); + CREATE UNIQUE INDEX y ON u (i); +COMMIT; +SELECT * FROM __Index ORDER BY TableName, ColumnName, Name; +|sTableName, sColumnName, sName, bIsUnique +[t i x false] +[t id() id false] +[u i y true] +[u t z false] + +-- 635 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (i); + CREATE INDEX id ON t (id()); + CREATE TABLE u (b bool, i bigint, t time, d duration); + CREATE INDEX z ON u (t); + CREATE UNIQUE INDEX y ON u (i); +COMMIT; +SELECT c.TableName, c.Ordinal, c.Name +FROM __Table AS t, __Column AS c +WHERE t.Name == "u" && t.Name == c.TableName +ORDER BY c.Ordinal; +|sc.TableName, lc.Ordinal, sc.Name +[u 1 b] +[u 2 i] +[u 3 t] +[u 4 d] + +-- 636 // https://github.com/cznic/ql/issues/36 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES (1, "test"); +COMMIT; +SELECT * FROM t WHERE s == "test"; +|li, ss +[1 test] + +-- 637 // https://github.com/cznic/ql/issues/36 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + INSERT INTO t VALUES (1, "test"); + CREATE INDEX idx_s ON t (s); +COMMIT; +SELECT * FROM t WHERE s == "test"; +|li, ss +[1 test] + +-- 638 // https://github.com/cznic/ql/issues/37 +BEGIN TRANSACTION; + CREATE TABLE artist (id int64, name string); + CREATE TABLE data_types (id int64, _uint int64, _uint8 int64, _uint16 + int64, _uint32 int64, _uint64 int64, _int int64, _int8 int64, + _int16 int64, _int32 int64, _int64 int64, _float32 float32, + _float64 float64, _bool bool, _string string, _date time, _time + time); +COMMIT; +SELECT * FROM __Table ORDER BY Name; // Must sort, map range is not deterministic. +|sName, sSchema +[artist CREATE TABLE artist (id int64, name string);] +[data_types CREATE TABLE data_types (id int64, _uint int64, _uint8 int64, _uint16 int64, _uint32 int64, _uint64 int64, _int int64, _int8 int64, _int16 int64, _int32 int64, _int64 int64, _float32 float32, _float64 float64, _bool bool, _string string, _date time, _time time);] + +-- 639 // https://github.com/cznic/ql/issues/37 +BEGIN TRANSACTION; + CREATE TABLE artist (id int64, name string); + CREATE TABLE data_types (id int64, _uint int64, _uint8 int64, _uint16 + int64, _uint32 int64, _uint64 int64, _int int64, _int8 int64, + _int16 int64, _int32 int64, _int64 int64, _float32 float32, + _float64 float64, _bool bool, _string string, _date time, _time + time); +COMMIT; +SELECT * FROM __Table WHERE Name == "artist"; +|sName, sSchema +[artist CREATE TABLE artist (id int64, name string);] + +-- 640 // https://github.com/cznic/ql/issues/37 +BEGIN TRANSACTION; + CREATE TABLE artist (id int64, name string); + CREATE TABLE data_types (id int64, _uint int64, _uint8 int64, _uint16 + int64, _uint32 int64, _uint64 int64, _int int64, _int8 int64, + _int16 int64, _int32 int64, _int64 int64, _float32 float32, + _float64 float64, _bool bool, _string string, _date time, _time + time); +COMMIT; +SELECT * FROM __Column ORDER BY TableName, Ordinal; +|sTableName, lOrdinal, sName, sType +[artist 1 id int64] +[artist 2 name string] +[data_types 1 id int64] +[data_types 2 _uint int64] +[data_types 3 _uint8 int64] +[data_types 4 _uint16 int64] +[data_types 5 _uint32 int64] +[data_types 6 _uint64 int64] +[data_types 7 _int int64] +[data_types 8 _int8 int64] +[data_types 9 _int16 int64] +[data_types 10 _int32 int64] +[data_types 11 _int64 int64] +[data_types 12 _float32 float32] +[data_types 13 _float64 float64] +[data_types 14 _bool bool] +[data_types 15 _string string] +[data_types 16 _date time] +[data_types 17 _time time] + +-- 641 // https://github.com/cznic/ql/issues/37 +BEGIN TRANSACTION; + CREATE TABLE artist (id int64, name string); + CREATE TABLE data_types (id int64, _uint int64, _uint8 int64, _uint16 + int64, _uint32 int64, _uint64 int64, _int int64, _int8 int64, + _int16 int64, _int32 int64, _int64 int64, _float32 float32, + _float64 float64, _bool bool, _string string, _date time, _time + time); +COMMIT; +SELECT * FROM __Column WHERE TableName == "artist" ORDER BY TableName, Ordinal; +|sTableName, lOrdinal, sName, sType +[artist 1 id int64] +[artist 2 name string] + +-- 642 +BEGIN TRANSACTION; + CREATE TABLE t (i int, j int, k int); + INSERT INTO t VALUES + (1, 2, 3), + (4, 5, 6); + CREATE TABLE u (x int, y int, z int); + INSERT INTO u VALUES + (10, 20, 30), + (40, 50, 60); +COMMIT; +SELECT * FROM t, u WHERE u.y < 60 && t.k < 7; +|lt.i, lt.j, lt.k, lu.x, lu.y, lu.z +[4 5 6 40 50 60] +[4 5 6 10 20 30] +[1 2 3 40 50 60] +[1 2 3 10 20 30] + +-- 643 // order -> xk used +BEGIN TRANSACTION; + CREATE TABLE t (i int, j int, k int); + CREATE INDEX xk ON t (k); + INSERT INTO t VALUES + (1, 2, 3), + (4, 5, 6); + CREATE TABLE u (x int, y int, z int); + INSERT INTO u VALUES + (10, 20, 30), + (40, 50, 60); +COMMIT; +SELECT * FROM t, u WHERE u.y < 60 && t.k < 7; +|lt.i, lt.j, lt.k, lu.x, lu.y, lu.z +[1 2 3 40 50 60] +[1 2 3 10 20 30] +[4 5 6 40 50 60] +[4 5 6 10 20 30] + +-- 644 // order -> xy used +BEGIN TRANSACTION; + CREATE TABLE t (i int, j int, k int); + INSERT INTO t VALUES + (1, 2, 3), + (4, 5, 6); + CREATE TABLE u (x int, y int, z int); + CREATE INDEX xy ON u (y); + INSERT INTO u VALUES + (10, 20, 30), + (40, 50, 60); +COMMIT; +SELECT * FROM t, u WHERE u.y < 60 && t.k < 7; +|lt.i, lt.j, lt.k, lu.x, lu.y, lu.z +[4 5 6 10 20 30] +[4 5 6 40 50 60] +[1 2 3 10 20 30] +[1 2 3 40 50 60] + +-- 645 // order -> both xk and xy used +BEGIN TRANSACTION; + CREATE TABLE t (i int, j int, k int); + CREATE INDEX xk ON t (k); + INSERT INTO t VALUES + (1, 2, 3), + (4, 5, 6); + CREATE TABLE u (x int, y int, z int); + CREATE INDEX xy ON u (y); + INSERT INTO u VALUES + (10, 20, 30), + (40, 50, 60); +COMMIT; +SELECT * FROM t, u WHERE u.y < 60 && t.k < 7; +|lt.i, lt.j, lt.k, lu.x, lu.y, lu.z +[1 2 3 10 20 30] +[1 2 3 40 50 60] +[4 5 6 10 20 30] +[4 5 6 40 50 60] + +-- 646 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +SELECT * FROM t OFFSET -1; // no rows -> not evaluated +|?i + +-- 647 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +SELECT * FROM t OFFSET 0; // no rows -> not evaluated +|?i + +-- 648 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +SELECT * FROM t OFFSET 1; // no rows -> not evaluated +|?i + +-- 649 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() OFFSET -1; +||invalid .* -1 .*must.* non-negative + +-- 650 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() OFFSET 0; +|li +[42] +[24] + +-- 651 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() OFFSET 1; +|li +[24] + +-- 652 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() OFFSET 2; +|?i + +-- 653 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT -1; // no rows -> not evaluated +|?i + +-- 654 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 0; // no rows -> not evaluated +|?i + +-- 655 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 1; // no rows -> not evaluated +|?i + +-- 656 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT -1; +||invalid .* -1 .*must.* non-negative + +-- 657 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 0; +|?i + +-- 658 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 1; +|li +[42] + +-- 659 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 2; +|li +[42] +[24] + +-- 660 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 3; +|li +[42] +[24] + +-- 661 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 0 OFFSET 0; +|?i + +-- 662 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 0 OFFSET 1; +|?i + +-- 663 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 0 OFFSET 2; +|?i + +-- 664 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 0 OFFSET 3; +|?i + +-- 665 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 1 OFFSET 0; +|li +[42] + +-- 666 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 1 OFFSET 1; +|li +[24] + +-- 667 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 1 OFFSET 2; +|?i + +-- 668 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 1 OFFSET 3; +|?i + +-- 669 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 2 OFFSET 0; +|li +[42] +[24] + +-- 670 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 2 OFFSET 1; +|li +[24] + +-- 671 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 2 OFFSET 2; +|?i + +-- 672 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 2 OFFSET 3; +|?i + +-- 673 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 3 OFFSET 0; +|li +[42] +[24] + +-- 674 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 3 OFFSET 1; +|li +[24] + +-- 675 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 3 OFFSET 2; +|?i + +-- 676 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(42), (24); +COMMIT; +SELECT * FROM t ORDER BY id() LIMIT 3 OFFSET 3; +|?i + +-- 677 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + CREATE TABLE u (i int); + INSERT INTO u VALUES(10), (20), (30); +COMMIT; +SELECT * FROM + (SELECT * FROM t ORDER BY i LIMIT 2 OFFSET 1;) AS a, + (SELECT * FROM u ORDER BY i) AS b, +ORDER BY a.i, b.i; +|la.i, lb.i +[2 10] +[2 20] +[2 30] +[3 10] +[3 20] +[3 30] + +-- 678 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + CREATE TABLE u (i int); + INSERT INTO u VALUES(10), (20), (30); +COMMIT; +SELECT * FROM + (SELECT * FROM t ORDER BY i LIMIT 2 OFFSET 1;) AS a, + (SELECT * FROM u ORDER BY i OFFSET 1) AS b, +ORDER BY a.i, b.i; +|la.i, lb.i +[2 20] +[2 30] +[3 20] +[3 30] + +-- 679 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + CREATE TABLE u (i int); + INSERT INTO u VALUES(10), (20), (30); +COMMIT; +SELECT * FROM + (SELECT * FROM t ORDER BY i LIMIT 2 OFFSET 1;) AS a, + (SELECT * FROM u ORDER BY i LIMIT 1) AS b, +ORDER BY a.i, b.i; +|la.i, lb.i +[2 10] +[3 10] + +-- 680 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + CREATE TABLE u (i int); + INSERT INTO u VALUES(10), (20), (30); +COMMIT; +SELECT * FROM + (SELECT * FROM t ORDER BY i LIMIT 2 OFFSET 1;) AS a, + (SELECT * FROM u ORDER BY i LIMIT 1 OFFSET 1) AS b, +ORDER BY a.i, b.i; +|la.i, lb.i +[2 20] +[3 20] + +-- 681 // https://github.com/cznic/ql/issues/41 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + CREATE TABLE u (i int); + INSERT INTO u VALUES(10), (20), (30); +COMMIT; +SELECT * FROM + (SELECT * FROM t ORDER BY i LIMIT 2 OFFSET 1;) AS a, + (SELECT * FROM u ORDER BY i LIMIT 1 OFFSET 1) AS b, +ORDER BY a.i, b.i +LIMIT 1; +|la.i, lb.i +[2 20] + +-- 682 // https://github.com/cznic/ql/issues/42 +BEGIN TRANSACTION; + DROP TABLE IF EXISTS fibonacci; + CREATE TABLE fibonacci( + input int, + output int + ); +COMMIT; + +BEGIN TRANSACTION; + INSERT INTO fibonacci (input, output) VALUES (0, 0); + INSERT INTO fibonacci (input, output) VALUES (1, 1); + INSERT INTO fibonacci (input, output) VALUES (2, 1); + INSERT INTO fibonacci (input, output) VALUES (3, 2); + INSERT INTO fibonacci (input, output) VALUES (4, 3); + INSERT INTO fibonacci (input, output) VALUES (5, 5); + INSERT INTO fibonacci (input, output) VALUES (6, 8); + INSERT INTO fibonacci (input, output) VALUES (7, 13); + INSERT INTO fibonacci (input, output) VALUES (8, 21); + INSERT INTO fibonacci (input, output) VALUES (9, 34); +COMMIT; + +--' Should print 4. +SELECT count(1) AS total FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3; +|ltotal +[4] + +-- 683 // https://github.com/cznic/ql/issues/42 +BEGIN TRANSACTION; + DROP TABLE IF EXISTS fibonacci; + CREATE TABLE fibonacci( + input int, + output int + ); +COMMIT; + +BEGIN TRANSACTION; + INSERT INTO fibonacci (input, output) VALUES (0, 0); + INSERT INTO fibonacci (input, output) VALUES (1, 1); + INSERT INTO fibonacci (input, output) VALUES (2, 1); + INSERT INTO fibonacci (input, output) VALUES (3, 2); + INSERT INTO fibonacci (input, output) VALUES (4, 3); + INSERT INTO fibonacci (input, output) VALUES (5, 5); + INSERT INTO fibonacci (input, output) VALUES (6, 8); + INSERT INTO fibonacci (input, output) VALUES (7, 13); + INSERT INTO fibonacci (input, output) VALUES (8, 21); + INSERT INTO fibonacci (input, output) VALUES (9, 34); +COMMIT; + +--' Should output (6, 8) (5, 5). +SELECT * FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3 ORDER BY input DESC LIMIT 2 OFFSET 1; +|linput, loutput +[6 8] +[5 5] + +-- 684 // https://github.com/cznic/ql/issues/42 +BEGIN TRANSACTION; + DROP TABLE IF EXISTS fibonacci; + CREATE TABLE fibonacci( + input int, + output int + ); +COMMIT; + +BEGIN TRANSACTION; + INSERT INTO fibonacci (input, output) VALUES (0, 0); + INSERT INTO fibonacci (input, output) VALUES (1, 1); + INSERT INTO fibonacci (input, output) VALUES (2, 1); + INSERT INTO fibonacci (input, output) VALUES (3, 2); + INSERT INTO fibonacci (input, output) VALUES (4, 3); + INSERT INTO fibonacci (input, output) VALUES (5, 5); + INSERT INTO fibonacci (input, output) VALUES (6, 8); + INSERT INTO fibonacci (input, output) VALUES (7, 13); + INSERT INTO fibonacci (input, output) VALUES (8, 21); + INSERT INTO fibonacci (input, output) VALUES (9, 34); + --' Let's delete 4 rows. + // Delete where input == 5, input == 6, input == 7 or input == 3 + DELETE FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3; +COMMIT; +SELECT * FROM fibonacci ORDER BY input; +|linput, loutput +[0 0] +[1 1] +[2 1] +[4 3] +[8 21] +[9 34] + +-- 685 // https://github.com/cznic/ql/issues/42 +BEGIN TRANSACTION; + DROP TABLE IF EXISTS fibonacci; + CREATE TABLE fibonacci( + input int, + output int + ); +COMMIT; + +BEGIN TRANSACTION; + INSERT INTO fibonacci (input, output) VALUES (0, 0); + INSERT INTO fibonacci (input, output) VALUES (1, 1); + INSERT INTO fibonacci (input, output) VALUES (2, 1); + INSERT INTO fibonacci (input, output) VALUES (3, 2); + INSERT INTO fibonacci (input, output) VALUES (4, 3); + INSERT INTO fibonacci (input, output) VALUES (5, 5); + INSERT INTO fibonacci (input, output) VALUES (6, 8); + INSERT INTO fibonacci (input, output) VALUES (7, 13); + INSERT INTO fibonacci (input, output) VALUES (8, 21); + INSERT INTO fibonacci (input, output) VALUES (9, 34); +COMMIT; +--' Let's delete 4 rows. +BEGIN TRANSACTION; + // Delete where input == 5, input == 6, input == 7 or input == 3 + DELETE FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3; +COMMIT; +SELECT * FROM fibonacci ORDER BY input; +|linput, loutput +[0 0] +[1 1] +[2 1] +[4 3] +[8 21] +[9 34] + +-- 686 // https://github.com/cznic/ql/issues/42 +BEGIN TRANSACTION; + DROP TABLE IF EXISTS fibonacci; + CREATE TABLE fibonacci( + input int, + output int + ); +COMMIT; + +BEGIN TRANSACTION; + INSERT INTO fibonacci (input, output) VALUES (0, 0); + INSERT INTO fibonacci (input, output) VALUES (1, 1); + INSERT INTO fibonacci (input, output) VALUES (2, 1); + INSERT INTO fibonacci (input, output) VALUES (3, 2); + INSERT INTO fibonacci (input, output) VALUES (4, 3); + INSERT INTO fibonacci (input, output) VALUES (5, 5); + INSERT INTO fibonacci (input, output) VALUES (6, 8); + INSERT INTO fibonacci (input, output) VALUES (7, 13); + INSERT INTO fibonacci (input, output) VALUES (8, 21); + INSERT INTO fibonacci (input, output) VALUES (9, 34); + --' Let's delete 4 rows. + // Delete where input == 5, input == 6, input == 7 or input == 3 + DELETE FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3; +COMMIT; +--' Try to count the rows we've just deleted, using the very same condition. Result is 1, should be 0. +SELECT count() AS total FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3; +|ltotal +[0] + +-- 687 // https://github.com/cznic/ql/issues/42 +BEGIN TRANSACTION; + DROP TABLE IF EXISTS fibonacci; + CREATE TABLE fibonacci( + input int, + output int + ); +COMMIT; + +BEGIN TRANSACTION; + INSERT INTO fibonacci (input, output) VALUES (0, 0); + INSERT INTO fibonacci (input, output) VALUES (1, 1); + INSERT INTO fibonacci (input, output) VALUES (2, 1); + INSERT INTO fibonacci (input, output) VALUES (3, 2); + INSERT INTO fibonacci (input, output) VALUES (4, 3); + INSERT INTO fibonacci (input, output) VALUES (5, 5); + INSERT INTO fibonacci (input, output) VALUES (6, 8); + INSERT INTO fibonacci (input, output) VALUES (7, 13); + INSERT INTO fibonacci (input, output) VALUES (8, 21); + INSERT INTO fibonacci (input, output) VALUES (9, 34); +COMMIT; +BEGIN TRANSACTION; + --' Let's delete 4 rows. + // Delete where input == 5, input == 6, input == 7 or input == 3 + DELETE FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3; +COMMIT; +--' Try to count the rows we've just deleted, using the very same condition. Result is 1, should be 0. +SELECT count() AS total FROM fibonacci WHERE input >= 5 && input <= 7 ⩖ input == 3; +|ltotal +[0] + +-- 688 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1); + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t ORDER BY i; +|?i + +-- 689 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2); + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[2] + +-- 690 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2); + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] + +-- 691 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[2] +[3] + +-- 692 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[3] + +-- 693 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3); + DELETE FROM t WHERE i == 3; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[2] + +-- 694 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[2] +[3] +[4] + +-- 695 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[3] +[4] + +-- 696 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 3; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[2] +[4] + +-- 697 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 4; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[2] +[3] + +-- 698 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 1; + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[3] +[4] + +-- 699 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 2; + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[3] +[4] + +-- 700 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 2; + DELETE FROM t WHERE i == 3; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[4] + +-- 701 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 3; + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[4] + +-- 702 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 3; + DELETE FROM t WHERE i == 4; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[2] + +-- 703 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 4; + DELETE FROM t WHERE i == 3; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[2] + +-- 704 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 1; + DELETE FROM t WHERE i == 3; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[2] +[4] + +-- 705 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 3; + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[2] +[4] + +-- 706 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 1; + DELETE FROM t WHERE i == 4; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[2] +[3] + +-- 707 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 4; + DELETE FROM t WHERE i == 1; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[2] +[3] + +-- 708 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 2; + DELETE FROM t WHERE i == 4; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[3] + +-- 709 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES(1), (2), (3), (4); + DELETE FROM t WHERE i == 4; + DELETE FROM t WHERE i == 2; +COMMIT; +SELECT * FROM t ORDER BY i; +|li +[1] +[3] + +-- 710 // https://github.com/cznic/ql/issues/43 +SELECT Name, Unique FROM __Index; +||syntax error + +-- 711 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); +COMMIT; +SELECT s FROM t; +|ss +[foo] +[bar] + +-- 712 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); +COMMIT; +SELECT * FROM x; +|sx +[bar] +[foo] + +-- 713 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (s); + CREATE INDEX x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); +COMMIT; +SELECT * FROM x; +||already + +-- 714 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); +COMMIT; +SELECT * FROM x; +|sx +[bar] +[foo] + +-- 715 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX x ON t (s); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); +COMMIT; +SELECT * FROM x; +|sx +[bar] +[foo] + +-- 716 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX IF NOT EXISTS x ON t (s); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); +COMMIT; +SELECT s FROM t WHERE s != "z"; +|ss +[foo] +[bar] + +-- 717 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); +COMMIT; +SELECT s FROM t WHERE s < "z"; +|ss +[bar] +[foo] + +-- 718 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); + DROP INDEX x; +COMMIT; +SELECT s FROM t WHERE s < "z"; +|ss +[foo] +[bar] + +-- 719 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); + DROP INDEX x; + DROP INDEX x; +COMMIT; +SELECT s FROM t WHERE s < "z"; +||does not exist + +-- 720 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); + DROP INDEX IF EXISTS x; + DROP INDEX x; +COMMIT; +SELECT s FROM t WHERE s < "z"; +||does not exist + +-- 721 +BEGIN TRANSACTION; + CREATE TABLE t (i int, s string); + CREATE INDEX IF NOT EXISTS x ON t (s); + INSERT INTO t VALUES (1, "bar"), (2, "foo"); + DROP INDEX x; + DROP INDEX IF EXISTS x; +COMMIT; +SELECT s FROM t WHERE s < "z"; +|ss +[foo] +[bar] + +-- 722 +BEGIN TRANSACTION; + CREATE TABLE t (p string, c blob); + CREATE UNIQUE INDEX x ON t (p); + INSERT INTO t VALUES + ("empty", blob("")), + ; +COMMIT; +SELECT p, string(c) FROM t; +|sp, s +[empty ] + +-- 723 +BEGIN TRANSACTION; + CREATE TABLE t (p string, c blob); + CREATE INDEX x ON t (p); + INSERT INTO t VALUES + ("empty", blob("")), + ; +COMMIT; +BEGIN TRANSACTION; + DELETE FROM t WHERE p == "empty"; +COMMIT; +SELECT p, string(c) FROM t; +|?p, ? + +-- 724 +BEGIN TRANSACTION; + CREATE TABLE t (p string, c blob); + CREATE UNIQUE INDEX x ON t (p); + INSERT INTO t VALUES + ("empty", blob("")), + ; +COMMIT; +BEGIN TRANSACTION; + DELETE FROM t WHERE p == "empty"; +COMMIT; +SELECT p, string(c) FROM t; +|?p, ? + +-- S 725 +BEGIN TRANSACTION; + UPDATE none SET + DepartmentID = DepartmentID+1000, + WHERE DepartmentID == 33; +COMMIT; +SELECT * FROM employee; +||table.*not.*exist + +-- S 726 +BEGIN TRANSACTION; + UPDATE employee SET + FirstName = "John" + WHERE DepartmentID == 33; +COMMIT; +SELECT * FROM employee; +||unknown.*FirstName + +-- S 727 +BEGIN TRANSACTION; + UPDATE employee SET + DepartmentID = DepartmentID+1000, + WHERE DepartmentID == 33; +COMMIT; +SELECT * FROM employee +ORDER BY LastName; +|sLastName, lDepartmentID +[Heisenberg 1033] +[John ] +[Jones 1033] +[Rafferty 31] +[Robinson 34] +[Smith 34] + +-- 728 // https://github.com/cznic/ql/issues/49 +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS t (username string, departname string, created time, detail_id int, height float64, avatar blob, is_man bool); + CREATE UNIQUE INDEX UQE_userinfo_username ON t (username); + INSERT INTO t (username, departname, created, detail_id, height, avatar, is_man) VALUES ( + "xiaolunwen", + "dev", + now(), + 1, + 1.78, + blob("012"), + true, + ); + DELETE FROM t WHERE id() == 1; +COMMIT; +SELECT * FROM t; +|?username, ?departname, ?created, ?detail_id, ?height, ?avatar, ?is_man + +-- 729 // https://github.com/cznic/ql/issues/49 +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS t (username string, departname string, created time, detail_id int, height float64, avatar blob, is_man bool); + CREATE UNIQUE INDEX UQE_userinfo_username ON t (username); + INSERT INTO t (username, departname, created, detail_id, height, avatar, is_man) VALUES ( + "xiaolunwen", + "dev", + now(), + 1, + 1.78, + __testBlob(256), + true, + ); + DELETE FROM t WHERE id() == 1; +COMMIT; +SELECT * FROM t; +|?username, ?departname, ?created, ?detail_id, ?height, ?avatar, ?is_man + +-- 730 // https://github.com/cznic/ql/issues/49 +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS t (username string, departname string, created time, detail_id int, height float64, avatar blob, is_man bool); + CREATE UNIQUE INDEX UQE_userinfo_username ON t (username); + INSERT INTO t (username, departname, created, detail_id, height, avatar, is_man) VALUES ( + "xiaolunwen", + "dev", + now(), + 1, + 1.78, + __testBlob(1<<16), + true, + ); + DELETE FROM t WHERE id() == 1; +COMMIT; +SELECT * FROM t; +|?username, ?departname, ?created, ?detail_id, ?height, ?avatar, ?is_man + +-- 731 // https://github.com/cznic/ql/issues/49 +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS t (username string, departname string, created time, detail_id int, height float64, avatar blob, is_man bool); + CREATE UNIQUE INDEX UQE_userinfo_username ON t (username); + INSERT INTO t (username, departname, created, detail_id, height, avatar, is_man) VALUES ( + "xiaolunwen", + "dev", + now(), + 1, + 1.78, + __testBlob(1<<20), + true, + ); + DELETE FROM t WHERE id() == 1; +COMMIT; +SELECT * FROM t; +|?username, ?departname, ?created, ?detail_id, ?height, ?avatar, ?is_man + +-- 732 // https://github.com/cznic/ql/issues/49 +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS t (username string, departname string, created time, detail_id int, height float64, avatar blob, is_man bool); + CREATE UNIQUE INDEX UQE_userinfo_username ON t (username); + INSERT INTO t (username, departname, created, detail_id, height, avatar, is_man) VALUES ( + "xiaolunwen", + "dev", + now(), + 1, + 1.78, + __testBlob(1<<20), + true, + ), ( + "2xiaolunwen", + "2dev", + now(), + 2, + 2.78, + __testBlob(1<<21), + true, + ); + DELETE FROM t WHERE id() == 1; +COMMIT; +SELECT id() == 2, username == "2xiaolunwen", len(string(avatar)) == 1<<21 FROM t; +|b, b, b +[true true true] + +-- 733 // https://github.com/cznic/ql/issues/49 +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS t (username string, departname string, created time, detail_id int, height float64, avatar blob, is_man bool); + CREATE UNIQUE INDEX UQE_userinfo_username ON t (username); + INSERT INTO t (username, departname, created, detail_id, height, avatar, is_man) VALUES ( + "xiaolunwen", + "dev", + now(), + 1, + 1.78, + __testBlob(1<<20), + true, + ), ( + "2xiaolunwen", + "2dev", + now(), + 2, + 2.78, + __testBlob(1<<21), + true, + ); + DELETE FROM t WHERE id() == 2; +COMMIT; +SELECT id() == 1, username == "xiaolunwen", len(string(avatar)) == 1<<20 FROM t; +|b, b, b +[true true true] + +-- 734 // https://github.com/cznic/ql/issues/51 +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS no_id_user (user string, remain int, total int); + CREATE UNIQUE INDEX UQE_no_id_user_user ON no_id_user (user); + DELETE FROM no_id_user WHERE user == "xlw"; + INSERT INTO no_id_user (user, remain, total) VALUES ("xlw", 20, 100); +COMMIT; +SELECT user, remain, total FROM no_id_user WHERE user == "xlw" LIMIT 1; +|suser, lremain, ltotal +[xlw 20 100] + +-- 735 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + INSERT INTO t VALUES (1), (2), (3), (4), (5), (6); +COMMIT; +SELECT * FROM t WHERE id() < 4; // reverse order -> no index used +|li +[3] +[2] +[1] + +-- 736 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES (1), (2), (3), (4), (5), (6); +COMMIT; +SELECT * FROM t WHERE id() < 4; // ordered -> index is used +|li +[1] +[2] +[3] + +-- 737 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES (1), (2), (3), (4), (5), (6); +COMMIT; +SELECT * FROM t WHERE id() <= 4; // ordered -> index is used +|li +[1] +[2] +[3] +[4] + +-- 738 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES (1), (2), (3), (4), (5), (6); +COMMIT; +SELECT * FROM t WHERE id() == 4; +|li +[4] + +-- 739 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES (1), (2), (3), (4), (5), (6); +COMMIT; +SELECT * FROM t WHERE id() >= 4; // ordered -> index is used +|li +[4] +[5] +[6] + +-- 740 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (id()); + INSERT INTO t VALUES (1), (2), (3), (4), (5), (6); +COMMIT; +SELECT * FROM t WHERE id() > 4; +|li +[6] +[5] + +-- 741 +BEGIN TRANSACTION; + CREATE TABLE t (i int); + CREATE INDEX x ON t (id()); + CREATE TABLE u (i int); + CREATE INDEX y ON u (i); + INSERT INTO t VALUES (1), (2), (3), (4), (5), (6); + INSERT INTO u VALUES (10), (20), (30), (40), (50), (60); +COMMIT; +SELECT t.ID, t.i, u.i FROM + (SELECT id() as ID, i FROM t WHERE id() < 4) AS t, + (SELECT * FROM u WHERE i < 40) AS u; // ordered -> both indices are used +|lt.ID, lt.i, lu.i +[1 1 10] +[1 1 20] +[1 1 30] +[2 2 10] +[2 2 20] +[2 2 30] +[3 3 10] +[3 3 20] +[3 3 30] + +-- 742 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (NULL), + ; +COMMIT; +SELECT max(t) as T FROM t; +|?T +[] + +-- 743 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (NULL), + (NULL), + ; +COMMIT; +SELECT max(t) as T FROM t; +|?T +[] + +-- 744 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (NULL), + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + ; +COMMIT; +SELECT max(t) as T FROM t; +|?T +[2014-08-08 14:05:11 +0000 UTC] + +-- 745 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + (NULL), + ; +COMMIT; +SELECT max(t) as T FROM t; +|?T +[2014-08-08 14:05:11 +0000 UTC] + +-- 746 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:12")), + ; +COMMIT; +SELECT max(t) as T FROM t; +|?T +[2014-08-08 14:05:12 +0000 UTC] + +-- 747 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:12")), + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + ; +COMMIT; +SELECT max(t) as T FROM t; +|?T +[2014-08-08 14:05:12 +0000 UTC] + +-- 748 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (NULL), + ; +COMMIT; +SELECT min(t) as T FROM t; +|?T +[] + +-- 749 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (NULL), + (NULL), + ; +COMMIT; +SELECT min(t) as T FROM t; +|?T +[] + +-- 750 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (NULL), + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + ; +COMMIT; +SELECT min(t) as T FROM t; +|?T +[2014-08-08 14:05:11 +0000 UTC] + +-- 751 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + (NULL), + ; +COMMIT; +SELECT min(t) as T FROM t; +|?T +[2014-08-08 14:05:11 +0000 UTC] + +-- 752 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:12")), + ; +COMMIT; +SELECT min(t) as T FROM t; +|?T +[2014-08-08 14:05:11 +0000 UTC] + +-- 753 // https://github.com/cznic/ql/pull/65 +BEGIN TRANSACTION; + CREATE TABLE t (t time); + INSERT INTO t VALUES + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:12")), + (parseTime("2006-01-02 15:04:05", "2014-08-08 14:05:11")), + ; +COMMIT; +SELECT min(t) as T FROM t; +|?T +[2014-08-08 14:05:11 +0000 UTC] + +-- 754 // https://github.com/cznic/ql/issues/68 +BEGIN TRANSACTION; + CREATE TABLE department (Name string); + INSERT INTO department (Name) VALUES ("small"), ("large"), ("medium"); + SELECT * FROM department; + ALTER TABLE department ADD score float; + SELECT * from department; + UPDATE department SET score=0 WHERE Name=="small"; +COMMIT; +SELECT * FROM department ORDER BY Name; +|sName, ?score +[large ] +[medium ] +[small 0] + +-- 755 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s LIKE "foo" +ORDER BY id(); +|l, ss +[1 seafood] +[2 A fool on the hill] +[5 foobar] + +-- 756 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE !(s LIKE "foo") +ORDER BY id(); +|l, ss +[4 barbaz] + +-- 757 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s LIKE "foo" IS NULL +ORDER BY id(); +|l, ?s +[3 ] + +-- 758 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s LIKE "foo" IS NOT NULL +ORDER BY id(); +|l, ss +[1 seafood] +[2 A fool on the hill] +[4 barbaz] +[5 foobar] + +-- 759 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s LIKE "bar" +ORDER BY id(); +|l, ss +[4 barbaz] +[5 foobar] + +-- 760 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s LIKE "^bar" +ORDER BY id(); +|l, ss +[4 barbaz] + +-- 761 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s LIKE "bar$" +ORDER BY id(); +|l, ss +[5 foobar] + +-- 762 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s LIKE "bar"+"$" +ORDER BY id(); +|l, ss +[5 foobar] + +-- 763 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s+"qux" LIKE "qux"+"$" +ORDER BY id(); +|l, ss +[1 seafood] +[2 A fool on the hill] +[4 barbaz] +[5 foobar] + +-- 764 +BEGIN TRANSACTION; + CREATE TABLE t (s string); + INSERT INTO t VALUES + ("seafood"), + ("A fool on the hill"), + (NULL), + ("barbaz"), + ("foobar"), + ; +COMMIT; +SELECT id(), s +FROM t +WHERE s+"quxx" LIKE "qux"+"$" +ORDER BY id(); +|?, ?s + +-- 765 // https://github.com/cznic/ql/issues/75 +BEGIN TRANSACTION; + CREATE TABLE foo (i int); + INSERT INTO foo VALUES (10), (20); + CREATE TABLE bar (fooID int, s string); + INSERT INTO bar SELECT id(), "ten" FROM foo WHERE i == 10; + INSERT INTO bar SELECT id(), "twenty" FROM foo WHERE i == 20; +COMMIT; +SELECT * +FROM + (SELECT id() AS ID, i FROM foo) AS foo, + bar +WHERE bar.fooID == foo.ID +ORDER BY foo.ID; +|lfoo.ID, lfoo.i, lbar.fooID, sbar.s +[1 10 1 ten] +[2 20 2 twenty] + +-- 766 // https://github.com/cznic/ql/issues/75 +BEGIN TRANSACTION; + CREATE TABLE foo (i int); + INSERT INTO foo VALUES (10), (20); + CREATE TABLE bar (fooID int, s string); + INSERT INTO bar SELECT id(), "ten" FROM foo WHERE i == 10; + INSERT INTO bar SELECT id(), "twenty" FROM foo WHERE i == 20; +COMMIT; +SELECT * +FROM foo, bar +WHERE bar.fooID == id(foo) +ORDER BY id(foo); +|lfoo.i, lbar.fooID, sbar.s +[10 1 ten] +[20 2 twenty] + +-- 767 // https://github.com/cznic/ql/issues/81 +BEGIN TRANSACTION; + CREATE TABLE t (name string, mail string); + INSERT INTO t VALUES + ("a", "foo@example.com"), + ("b", "bar@example.com"), + ("c", "baz@example.com"), + ("d", "foo@example.com"), + ("e", "bar@example.com"), + ("f", "baz@example.com"), + ; +COMMIT; +SELECT * +FROM t +WHERE name == "b" AND mail == "bar@example.com"; +|sname, smail +[b bar@example.com] + +-- 768 // https://github.com/cznic/ql/issues/81 +BEGIN TRANSACTION; + CREATE TABLE t (name string, mail string); + INSERT INTO t VALUES + ("a", "foo@example.com"), + ("b", "bar@example.com"), + ("c", "baz@example.com"), + ("d", "foo@example.com"), + ("e", "bar@example.com"), + ("f", "baz@example.com"), + ; +COMMIT; +SELECT * +FROM t +WHERE name == "b" and mail == "bar@example.com"; +|sname, smail +[b bar@example.com] + +-- 769 // https://github.com/cznic/ql/issues/81 +BEGIN TRANSACTION; + CREATE TABLE t (name string, mail string); + INSERT INTO t VALUES + ("a", "foo@example.com"), + ("b", "bar@example.com"), + ("c", "baz@example.com"), + ("d", "foo@example.com"), + ("e", "bar@example.com"), + ("f", "baz@example.com"), + ; +COMMIT; +SELECT * +FROM t +WHERE name == "b" OR mail == "bar@example.com" +ORDER BY name; +|sname, smail +[b bar@example.com] +[e bar@example.com] + +-- 770 // https://github.com/cznic/ql/issues/81 +BEGIN TRANSACTION; + CREATE TABLE t (name string, mail string); + INSERT INTO t VALUES + ("a", "foo@example.com"), + ("b", "bar@example.com"), + ("c", "baz@example.com"), + ("d", "foo@example.com"), + ("e", "bar@example.com"), + ("f", "baz@example.com"), + ; +COMMIT; +SELECT * +FROM t +WHERE name == "b" or mail == "bar@example.com" +ORDER BY name; +|sname, smail +[b bar@example.com] +[e bar@example.com] + +-- 771 // https://github.com/cznic/ql/issues/72 +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; +COMMIT; +SELECT id(), i FROM tableA WHERE id() IN (SELECT idA FROM tableB) ORDER BY id(); +|l, li +[2 12] +[4 14] +[6 16] + +-- 772 // https://github.com/cznic/ql/issues/72 +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; +COMMIT; +SELECT id(), i FROM tableA WHERE id() NOT IN (SELECT idA FROM tableB) ORDER BY id(); +|l, li +[1 11] +[3 13] +[5 15] + +-- 773 // https://github.com/cznic/ql/issues/72 +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE id() IN (SELECT idA FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +|l, li +[1 11] +[3 13] +[5 15] + +-- 774 // https://github.com/cznic/ql/issues/72 +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE id() NOT IN (SELECT idA FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +|l, li +[2 12] +[4 14] +[6 16] + +-- 775 // https://github.com/cznic/ql/issues/72, coerce +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE 2 IN (SELECT idA FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +|?, ?i + +-- 776 // https://github.com/cznic/ql/issues/72, coerce +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE 2 NOT IN (SELECT idA FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +|l, li +[1 11] +[2 12] +[3 13] +[4 14] +[5 15] +[6 16] + +-- 777 // https://github.com/cznic/ql/issues/72, different types have zero set intersetion. +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE 3.14 IN (SELECT idA FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +|l, li +[1 11] +[2 12] +[3 13] +[4 14] +[5 15] +[6 16] + +-- 778 // https://github.com/cznic/ql/issues/72, different have zero set intersection but NOT makes the result true. +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE 3.14 NOT IN (SELECT idA FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +|?, ?i + +-- 779 // https://github.com/cznic/ql/issues/72, invalid field type +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA time); + INSERT INTO tableB + SELECT now() FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE 3.14 NOT IN (SELECT idA FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +||invalid field type + +-- 780 // https://github.com/cznic/ql/issues/72, too many fields +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int, name string); + INSERT INTO tableB + SELECT id(), "foo" FROM tableA WHERE i&1 == 0; + DELETE FROM tableA WHERE id() NOT IN (SELECT * FROM tableB); +COMMIT; +SELECT id(), i FROM tableA ORDER BY id(); +||mismatched field count + +-- 781 // https://github.com/cznic/ql/issues/72, some NULL +BEGIN TRANSACTION; + DROP TABLE IF EXISTS tableA; + DROP TABLE IF EXISTS tableB; +COMMIT; +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB VALUES(NULL); + INSERT INTO tableB + SELECT id() FROM tableA WHERE i&1 == 0; +COMMIT; +SELECT i FROM tableA WHERE id() IN (SELECT idA from tableB) ORDER BY id(); +|li +[12] +[14] +[16] + +-- 782 // https://github.com/cznic/ql/issues/72, all NULL +BEGIN TRANSACTION; + DROP TABLE IF EXISTS tableA; + DROP TABLE IF EXISTS tableB; +COMMIT; +BEGIN TRANSACTION; + CREATE TABLE tableA (i int); + INSERT INTO tableA VALUES + (11), + (12), + (13), + (14), + (15), + (16), + ; + CREATE TABLE tableB (idA int); + INSERT INTO tableB VALUES(NULL); +COMMIT; +SELECT i FROM tableA WHERE id() IN (SELECT idA from tableB) ORDER BY id(); +|?i diff --git a/Godeps/_workspace/src/github.com/cznic/sortutil/AUTHORS b/Godeps/_workspace/src/github.com/cznic/sortutil/AUTHORS new file mode 100644 index 00000000000..0078f5f5b6d --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/sortutil/AUTHORS @@ -0,0 +1,11 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/sortutil/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/sortutil/CONTRIBUTORS new file mode 100644 index 00000000000..9c9a5dd84c2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/sortutil/CONTRIBUTORS @@ -0,0 +1,10 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Gary Burd +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/sortutil/LICENSE b/Godeps/_workspace/src/github.com/cznic/sortutil/LICENSE new file mode 100644 index 00000000000..67983e0e618 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/sortutil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The sortutil Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/sortutil/Makefile b/Godeps/_workspace/src/github.com/cznic/sortutil/Makefile new file mode 100644 index 00000000000..b77748de93c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/sortutil/Makefile @@ -0,0 +1,35 @@ +# Copyright 2014 The sortutil Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +.PHONY: all clean editor later nuke todo + +grep=--include=*.go + +all: editor + go vet + golint . + make todo + +clean: + go clean + rm -f *~ + +editor: + go fmt + go test -i + go test + go build + +later: + @grep -n $(grep) LATER * || true + @grep -n $(grep) MAYBE * || true + +nuke: clean + go clean -i + +todo: + @grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * || true + @grep -nr $(grep) TODO * || true + @grep -nr $(grep) BUG * || true + @grep -nr $(grep) println * || true diff --git a/Godeps/_workspace/src/github.com/cznic/sortutil/README b/Godeps/_workspace/src/github.com/cznic/sortutil/README new file mode 100644 index 00000000000..84ef92ccd0c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/sortutil/README @@ -0,0 +1,4 @@ +Packages in this repository: + +Install: $ go get github.com/cznic/sortutil +Godocs: http://godoc.org/github.com/cznic/sortutil diff --git a/Godeps/_workspace/src/github.com/cznic/sortutil/all_test.go b/Godeps/_workspace/src/github.com/cznic/sortutil/all_test.go new file mode 100644 index 00000000000..1e3a010812c --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/sortutil/all_test.go @@ -0,0 +1,360 @@ +// Copyright 2014 The sortutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sortutil + +import ( + "fmt" + "math" + "path" + "runtime" + "sort" + "strings" + "testing" + + "github.com/cznic/mathutil" +) + +func dbg(s string, va ...interface{}) { + if s == "" { + s = strings.Repeat("%v ", len(va)) + } + _, fn, fl, _ := runtime.Caller(1) + fmt.Printf("dbg %s:%d: ", path.Base(fn), fl) + fmt.Printf(s, va...) + fmt.Println() +} + +func caller(s string, va ...interface{}) { + _, fn, fl, _ := runtime.Caller(2) + fmt.Printf("caller: %s:%d: ", path.Base(fn), fl) + fmt.Printf(s, va...) + fmt.Println() + _, fn, fl, _ = runtime.Caller(1) + fmt.Printf("\tcallee: %s:%d: ", path.Base(fn), fl) + fmt.Println() +} + +func use(...interface{}) {} + +func TestByteSlice(t *testing.T) { + const N = 1e4 + s := make(ByteSlice, N) + for i := range s { + s[i] = byte(i) ^ 0x55 + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchBytes(t *testing.T) { + const N = 1e1 + s := make(ByteSlice, N) + for i := range s { + s[i] = byte(2 * i) + } + if g, e := SearchBytes(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestFloat32Slice(t *testing.T) { + const N = 1e4 + s := make(Float32Slice, N) + for i := range s { + s[i] = float32(i ^ 0x55aa55aa) + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchFloat32s(t *testing.T) { + const N = 1e4 + s := make(Float32Slice, N) + for i := range s { + s[i] = float32(2 * i) + } + if g, e := SearchFloat32s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestInt8Slice(t *testing.T) { + const N = 1e4 + s := make(Int8Slice, N) + for i := range s { + s[i] = int8(i) ^ 0x55 + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchInt8s(t *testing.T) { + const N = 1e1 + s := make(Int8Slice, N) + for i := range s { + s[i] = int8(2 * i) + } + if g, e := SearchInt8s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestInt16Slice(t *testing.T) { + const N = 1e4 + s := make(Int16Slice, N) + for i := range s { + s[i] = int16(i) ^ 0x55aa + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchInt16s(t *testing.T) { + const N = 1e4 + s := make(Int16Slice, N) + for i := range s { + s[i] = int16(2 * i) + } + if g, e := SearchInt16s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestInt32Slice(t *testing.T) { + const N = 1e4 + s := make(Int32Slice, N) + for i := range s { + s[i] = int32(i) ^ 0x55aa55aa + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchInt32s(t *testing.T) { + const N = 1e4 + s := make(Int32Slice, N) + for i := range s { + s[i] = int32(2 * i) + } + if g, e := SearchInt32s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestInt64Slice(t *testing.T) { + const N = 1e4 + s := make(Int64Slice, N) + for i := range s { + s[i] = int64(i) ^ 0x55aa55aa55aa55aa + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchInt64s(t *testing.T) { + const N = 1e4 + s := make(Int64Slice, N) + for i := range s { + s[i] = int64(2 * i) + } + if g, e := SearchInt64s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestUintSlice(t *testing.T) { + const N = 1e4 + s := make(UintSlice, N) + for i := range s { + s[i] = uint(i) ^ 0x55aa55aa + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchUints(t *testing.T) { + const N = 1e4 + s := make(UintSlice, N) + for i := range s { + s[i] = uint(2 * i) + } + if g, e := SearchUints(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestUint16Slice(t *testing.T) { + const N = 1e4 + s := make(Uint16Slice, N) + for i := range s { + s[i] = uint16(i) ^ 0x55aa + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchUint16s(t *testing.T) { + const N = 1e4 + s := make(Uint16Slice, N) + for i := range s { + s[i] = uint16(2 * i) + } + if g, e := SearchUint16s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestUint32Slice(t *testing.T) { + const N = 1e4 + s := make(Uint32Slice, N) + for i := range s { + s[i] = uint32(i) ^ 0x55aa55aa + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchUint32s(t *testing.T) { + const N = 1e4 + s := make(Uint32Slice, N) + for i := range s { + s[i] = uint32(2 * i) + } + if g, e := SearchUint32s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestUint64Slice(t *testing.T) { + const N = 1e4 + s := make(Uint64Slice, N) + for i := range s { + s[i] = uint64(i) ^ 0x55aa55aa55aa55aa + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchUint64s(t *testing.T) { + const N = 1e4 + s := make(Uint64Slice, N) + for i := range s { + s[i] = uint64(2 * i) + } + if g, e := SearchUint64s(s, 12), 6; g != e { + t.Fatal(g, e) + } +} + +func TestRuneSlice(t *testing.T) { + const N = 1e4 + s := make(RuneSlice, N) + for i := range s { + s[i] = rune(i ^ 0x55aa55aa) + } + s.Sort() + if !sort.IsSorted(s) { + t.Fatal(false) + } +} + +func TestSearchRunes(t *testing.T) { + const N = 1e4 + s := make(RuneSlice, N) + for i := range s { + s[i] = rune(2 * i) + } + if g, e := SearchRunes(s, rune('\x0c')), 6; g != e { + t.Fatal(g, e) + } +} + +func dedupe(a []int) (r []int) { + a = append([]int(nil), a...) + if len(a) < 2 { + return a + } + + sort.Ints(a) + if a[0] < 0 { + panic("internal error") + } + + last := -1 + for _, v := range a { + if v != last { + r = append(r, v) + last = v + } + } + return r +} + +func TestDedup(t *testing.T) { + a := []int{} + n := Dedupe(sort.IntSlice(a)) + if g, e := n, 0; g != e { + t.Fatal(g, e) + } + + if g, e := len(a), 0; g != e { + t.Fatal(g, e) + } + + for c := 1; c <= 7; c++ { + in := make([]int, c) + lim := int(mathutil.ModPowUint32(uint32(c), uint32(c), math.MaxUint32)) + for n := 0; n < lim; n++ { + m := n + for i := range in { + in[i] = m % c + m /= c + } + in0 := append([]int(nil), in...) + out0 := dedupe(in) + n := Dedupe(sort.IntSlice(in)) + if g, e := n, len(out0); g != e { + t.Fatalf("n %d, exp %d, in0 %v, in %v, out0 %v", g, e, in0, in, out0) + } + + for i, v := range out0 { + if g, e := in[i], v; g != e { + t.Fatalf("n %d, in0 %v, in %v, out0 %v", n, in0, in, out0) + } + } + } + } +} + +func ExampleDedupe() { + a := []int{4, 1, 2, 1, 3, 4, 2} + fmt.Println(a[:Dedupe(sort.IntSlice(a))]) + + b := []string{"foo", "bar", "baz", "bar", "foo", "qux", "qux"} + fmt.Println(b[:Dedupe(sort.StringSlice(b))]) + // Output: + // [1 2 3 4] + // [bar baz foo qux] +} diff --git a/Godeps/_workspace/src/github.com/cznic/sortutil/sortutil.go b/Godeps/_workspace/src/github.com/cznic/sortutil/sortutil.go new file mode 100644 index 00000000000..c0ced542d39 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/sortutil/sortutil.go @@ -0,0 +1,227 @@ +// Copyright 2014 The sortutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sortutil provides utilities supplementing the standard 'sort' package. +package sortutil + +import "sort" + +// ByteSlice attaches the methods of sort.Interface to []byte, sorting in increasing order. +type ByteSlice []byte + +func (s ByteSlice) Len() int { return len(s) } +func (s ByteSlice) Less(i, j int) bool { return s[i] < s[j] } +func (s ByteSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s ByteSlice) Sort() { + sort.Sort(s) +} + +// SearchBytes searches for x in a sorted slice of bytes and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchBytes(a []byte, x byte) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Float32Slice attaches the methods of sort.Interface to []float32, sorting in increasing order. +type Float32Slice []float32 + +func (s Float32Slice) Len() int { return len(s) } +func (s Float32Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Float32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Float32Slice) Sort() { + sort.Sort(s) +} + +// SearchFloat32s searches for x in a sorted slice of float32 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchFloat32s(a []float32, x float32) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Int8Slice attaches the methods of sort.Interface to []int8, sorting in increasing order. +type Int8Slice []int8 + +func (s Int8Slice) Len() int { return len(s) } +func (s Int8Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Int8Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Int8Slice) Sort() { + sort.Sort(s) +} + +// SearchInt8s searches for x in a sorted slice of int8 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchInt8s(a []int8, x int8) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Int16Slice attaches the methods of sort.Interface to []int16, sorting in increasing order. +type Int16Slice []int16 + +func (s Int16Slice) Len() int { return len(s) } +func (s Int16Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Int16Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Int16Slice) Sort() { + sort.Sort(s) +} + +// SearchInt16s searches for x in a sorted slice of int16 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchInt16s(a []int16, x int16) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Int32Slice attaches the methods of sort.Interface to []int32, sorting in increasing order. +type Int32Slice []int32 + +func (s Int32Slice) Len() int { return len(s) } +func (s Int32Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Int32Slice) Sort() { + sort.Sort(s) +} + +// SearchInt32s searches for x in a sorted slice of int32 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchInt32s(a []int32, x int32) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Int64Slice attaches the methods of sort.Interface to []int64, sorting in increasing order. +type Int64Slice []int64 + +func (s Int64Slice) Len() int { return len(s) } +func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Int64Slice) Sort() { + sort.Sort(s) +} + +// SearchInt64s searches for x in a sorted slice of int64 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchInt64s(a []int64, x int64) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// UintSlice attaches the methods of sort.Interface to []uint, sorting in increasing order. +type UintSlice []uint + +func (s UintSlice) Len() int { return len(s) } +func (s UintSlice) Less(i, j int) bool { return s[i] < s[j] } +func (s UintSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s UintSlice) Sort() { + sort.Sort(s) +} + +// SearchUints searches for x in a sorted slice of uints and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchUints(a []uint, x uint) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Uint16Slice attaches the methods of sort.Interface to []uint16, sorting in increasing order. +type Uint16Slice []uint16 + +func (s Uint16Slice) Len() int { return len(s) } +func (s Uint16Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Uint16Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Uint16Slice) Sort() { + sort.Sort(s) +} + +// SearchUint16s searches for x in a sorted slice of uint16 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchUint16s(a []uint16, x uint16) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Uint32Slice attaches the methods of sort.Interface to []uint32, sorting in increasing order. +type Uint32Slice []uint32 + +func (s Uint32Slice) Len() int { return len(s) } +func (s Uint32Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Uint32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Uint32Slice) Sort() { + sort.Sort(s) +} + +// SearchUint32s searches for x in a sorted slice of uint32 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchUint32s(a []uint32, x uint32) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Uint64Slice attaches the methods of sort.Interface to []uint64, sorting in increasing order. +type Uint64Slice []uint64 + +func (s Uint64Slice) Len() int { return len(s) } +func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Uint64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s Uint64Slice) Sort() { + sort.Sort(s) +} + +// SearchUint64s searches for x in a sorted slice of uint64 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchUint64s(a []uint64, x uint64) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// RuneSlice attaches the methods of sort.Interface to []rune, sorting in increasing order. +type RuneSlice []rune + +func (s RuneSlice) Len() int { return len(s) } +func (s RuneSlice) Less(i, j int) bool { return s[i] < s[j] } +func (s RuneSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Sort is a convenience method. +func (s RuneSlice) Sort() { + sort.Sort(s) +} + +// SearchRunes searches for x in a sorted slice of uint64 and returns the index +// as specified by sort.Search. The slice must be sorted in ascending order. +func SearchRunes(a []rune, x rune) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +// Dedupe returns n, the number of distinct elements in data. The resulting +// elements are sorted in elements [0, n) or data[:n] for a slice. +func Dedupe(data sort.Interface) (n int) { + if n = data.Len(); n < 2 { + return n + } + + sort.Sort(data) + a, b := 0, 1 + for b < n { + if data.Less(a, b) { + a++ + if a != b { + data.Swap(a, b) + } + } + b++ + } + return a + 1 +} diff --git a/Godeps/_workspace/src/github.com/cznic/strutil/AUTHORS b/Godeps/_workspace/src/github.com/cznic/strutil/AUTHORS new file mode 100644 index 00000000000..d04c4501186 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/strutil/AUTHORS @@ -0,0 +1,12 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +CZ.NIC z.s.p.o. +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/strutil/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/strutil/CONTRIBUTORS new file mode 100644 index 00000000000..5e86f0607ce --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/strutil/CONTRIBUTORS @@ -0,0 +1,9 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> diff --git a/Godeps/_workspace/src/github.com/cznic/strutil/LICENSE b/Godeps/_workspace/src/github.com/cznic/strutil/LICENSE new file mode 100644 index 00000000000..2fdd92cf713 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/strutil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The strutil Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/strutil/Makefile b/Godeps/_workspace/src/github.com/cznic/strutil/Makefile new file mode 100644 index 00000000000..0cdf1cf8857 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/strutil/Makefile @@ -0,0 +1,37 @@ +.PHONY: all clean editor later nuke todo cover cpu + +grep=--include=*.go +target=foo.test + +all: editor + make todo + +clean: + go clean + rm -f *~ + +cover: + t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t + +cpu: $(target) + ./$< -noerr -test.cpuprofile cpu.out + go tool pprof --lines $< cpu.out + +editor: + go fmt + go test -i + go test + go build + +later: + @grep -n $(grep) LATER * || true + @grep -n $(grep) MAYBE * || true + +nuke: clean + go clean -i + +todo: + @grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * || true + @grep -nr $(grep) TODO * || true + @grep -nr $(grep) BUG * || true + @grep -nr $(grep) [^[:alpha:]]println * || true diff --git a/Godeps/_workspace/src/github.com/cznic/strutil/README b/Godeps/_workspace/src/github.com/cznic/strutil/README new file mode 100644 index 00000000000..a8652a3dc0a --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/strutil/README @@ -0,0 +1,8 @@ +This is a goinstall-able mirror of modified code already published at: +http://git.nic.cz/redmine/projects/gostrutil/repository + +Online godoc documentation for this package (should be) available at: +http://gopkgdoc.appspot.com/pkg/github.com/cznic/strutil + +Installation: +$ go get github.com/cznic/strutil diff --git a/Godeps/_workspace/src/github.com/cznic/strutil/all_test.go b/Godeps/_workspace/src/github.com/cznic/strutil/all_test.go new file mode 100644 index 00000000000..c987efcf4d4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/strutil/all_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2014 The sortutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strutil + +import ( + "bytes" + "github.com/cznic/mathutil" + "math" + "strings" + "testing" +) + +func TestBase64(t *testing.T) { + const max = 768 + r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + t.Fatal(err) + } + + bin := []byte{} + for i := 0; i < max; i++ { + bin = append(bin, byte(r.Next())) + cmp, err := Base64Decode(Base64Encode(bin)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(bin, cmp) { + t.Fatalf("a: % x\nb: % x", bin, cmp) + } + } +} + +func TestBase32Ext(t *testing.T) { + const max = 640 + r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true) + if err != nil { + t.Fatal(err) + } + + bin := []byte{} + for i := 0; i < max; i++ { + bin = append(bin, byte(r.Next())) + cmp, err := Base32ExtDecode(Base32ExtEncode(bin)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(bin, cmp) { + t.Fatalf("a: % x\nb: % x", bin, cmp) + } + } +} + +func TestFields(t *testing.T) { + p := []string{"", "\\", "|", "0", "1", "2"} + one := func(n int) string { + s := "" + for i := 0; i < 3; i++ { + s += p[n%len(p)] + n /= len(p) + } + return s + } + max := len(p) * len(p) * len(p) + var a [3]string + for x := 0; x < max; x++ { + a[0] = one(x) + for x := 0; x < max; x++ { + a[1] = one(x) + for x := 0; x < len(p)*len(p); x++ { + a[2] = one(x) + enc := JoinFields(a[:], "|") + dec := SplitFields(enc, "|") + if g, e := strings.Join(dec, ","), strings.Join(a[:], ","); g != e { + t.Fatal(g, e) + } + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/strutil/strutil.go b/Godeps/_workspace/src/github.com/cznic/strutil/strutil.go new file mode 100644 index 00000000000..ba83489df79 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/strutil/strutil.go @@ -0,0 +1,428 @@ +// Copyright (c) 2014 The sortutil Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package strutil collects utils supplemental to the standard strings package. +package strutil + +import ( + "bytes" + "encoding/base32" + "encoding/base64" + "fmt" + "io" + "strings" + "sync" +) + +// Base32ExtDecode decodes base32 extended (RFC 4648) text to binary data. +func Base32ExtDecode(text []byte) (data []byte, err error) { + n := base32.HexEncoding.DecodedLen(len(text)) + data = make([]byte, n) + decoder := base32.NewDecoder(base32.HexEncoding, bytes.NewBuffer(text)) + if n, err = decoder.Read(data); err != nil { + n = 0 + } + data = data[:n] + return +} + +// Base32ExtEncode encodes binary data to base32 extended (RFC 4648) encoded text. +func Base32ExtEncode(data []byte) (text []byte) { + n := base32.HexEncoding.EncodedLen(len(data)) + buf := bytes.NewBuffer(make([]byte, 0, n)) + encoder := base32.NewEncoder(base32.HexEncoding, buf) + encoder.Write(data) + encoder.Close() + if buf.Len() != n { + panic("internal error") + } + return buf.Bytes() +} + +// Base64Decode decodes base64 text to binary data. +func Base64Decode(text []byte) (data []byte, err error) { + n := base64.StdEncoding.DecodedLen(len(text)) + data = make([]byte, n) + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewBuffer(text)) + if n, err = decoder.Read(data); err != nil { + n = 0 + } + data = data[:n] + return +} + +// Base64Encode encodes binary data to base64 encoded text. +func Base64Encode(data []byte) (text []byte) { + n := base64.StdEncoding.EncodedLen(len(data)) + buf := bytes.NewBuffer(make([]byte, 0, n)) + encoder := base64.NewEncoder(base64.StdEncoding, buf) + encoder.Write(data) + encoder.Close() + if buf.Len() != n { + panic("internal error") + } + return buf.Bytes() +} + +// Formatter is an io.Writer extended by a fmt.Printf like function Format +type Formatter interface { + io.Writer + Format(format string, args ...interface{}) (n int, errno error) +} + +type indentFormatter struct { + io.Writer + indent []byte + indentLevel int + state int +} + +const ( + st0 = iota + stBOL + stPERC + stBOL_PERC +) + +// IndentFormatter returns a new Formatter which interprets %i and %u in the +// Format() format string as indent and undent commands. The commands can +// nest. The Formatter writes to io.Writer 'w' and inserts one 'indent' +// string per current indent level value. +// Behaviour of commands reaching negative indent levels is undefined. +// IndentFormatter(os.Stdout, "\t").Format("abc%d%%e%i\nx\ny\n%uz\n", 3) +// output: +// abc3%e +// x +// y +// z +// The Go quoted string literal form of the above is: +// "abc%%e\n\tx\n\tx\nz\n" +// The commands can be scattered between separate invocations of Format(), +// i.e. the formatter keeps track of the indent level and knows if it is +// positioned on start of a line and should emit indentation(s). +// The same output as above can be produced by e.g.: +// f := IndentFormatter(os.Stdout, " ") +// f.Format("abc%d%%e%i\nx\n", 3) +// f.Format("y\n%uz\n") +func IndentFormatter(w io.Writer, indent string) Formatter { + return &indentFormatter{w, []byte(indent), 0, stBOL} +} + +func (f *indentFormatter) format(flat bool, format string, args ...interface{}) (n int, errno error) { + buf := []byte{} + for i := 0; i < len(format); i++ { + c := format[i] + switch f.state { + case st0: + switch c { + case '\n': + cc := c + if flat && f.indentLevel != 0 { + cc = ' ' + } + buf = append(buf, cc) + f.state = stBOL + case '%': + f.state = stPERC + default: + buf = append(buf, c) + } + case stBOL: + switch c { + case '\n': + cc := c + if flat && f.indentLevel != 0 { + cc = ' ' + } + buf = append(buf, cc) + case '%': + f.state = stBOL_PERC + default: + if !flat { + for i := 0; i < f.indentLevel; i++ { + buf = append(buf, f.indent...) + } + } + buf = append(buf, c) + f.state = st0 + } + case stBOL_PERC: + switch c { + case 'i': + f.indentLevel++ + f.state = stBOL + case 'u': + f.indentLevel-- + f.state = stBOL + default: + if !flat { + for i := 0; i < f.indentLevel; i++ { + buf = append(buf, f.indent...) + } + } + buf = append(buf, '%', c) + f.state = st0 + } + case stPERC: + switch c { + case 'i': + f.indentLevel++ + f.state = st0 + case 'u': + f.indentLevel-- + f.state = st0 + default: + buf = append(buf, '%', c) + f.state = st0 + } + default: + panic("unexpected state") + } + } + switch f.state { + case stPERC, stBOL_PERC: + buf = append(buf, '%') + } + return f.Write([]byte(fmt.Sprintf(string(buf), args...))) +} + +func (f *indentFormatter) Format(format string, args ...interface{}) (n int, errno error) { + return f.format(false, format, args...) +} + +type flatFormatter indentFormatter + +// FlatFormatter returns a newly created Formatter with the same functionality as the one returned +// by IndentFormatter except it allows a newline in the 'format' string argument of Format +// to pass through iff indent level is currently zero. +// +// If indent level is non-zero then such new lines are changed to a space character. +// There is no indent string, the %i and %u format verbs are used solely to determine the indent level. +// +// The FlatFormatter is intended for flattening of normally nested structure textual representation to +// a one top level structure per line form. +// FlatFormatter(os.Stdout, " ").Format("abc%d%%e%i\nx\ny\n%uz\n", 3) +// output in the form of a Go quoted string literal: +// "abc3%%e x y z\n" +func FlatFormatter(w io.Writer) Formatter { + return (*flatFormatter)(IndentFormatter(w, "").(*indentFormatter)) +} + +func (f *flatFormatter) Format(format string, args ...interface{}) (n int, errno error) { + return (*indentFormatter)(f).format(true, format, args...) +} + +// Pool handles aligning of strings having equal values to the same string instance. +// Intended use is to conserve some memory e.g. where a large number of identically valued strings +// with non identical backing arrays may exists in several semantically distinct instances of some structs. +// Pool is *not* concurrent access safe. It doesn't handle common prefix/suffix aligning, +// e.g. having s1 == "abc" and s2 == "bc", s2 is not automatically aligned as s1[1:]. +type Pool struct { + pool map[string]string +} + +// NewPool returns a newly created Pool. +func NewPool() *Pool { + return &Pool{map[string]string{}} +} + +// Align returns a string with the same value as its argument. It guarantees that +// all aligned strings share a single instance in memory. +func (p *Pool) Align(s string) string { + if a, ok := p.pool[s]; ok { + return a + } + + s = StrPack(s) + p.pool[s] = s + return s +} + +// Count returns the number of items in the pool. +func (p *Pool) Count() int { + return len(p.pool) +} + +// GoPool is a concurrent access safe version of Pool. +type GoPool struct { + pool map[string]string + rwm *sync.RWMutex +} + +// NewGoPool returns a newly created GoPool. +func NewGoPool() (p *GoPool) { + return &GoPool{map[string]string{}, &sync.RWMutex{}} +} + +// Align returns a string with the same value as its argument. It guarantees that +// all aligned strings share a single instance in memory. +func (p *GoPool) Align(s string) (y string) { + if s != "" { + p.rwm.RLock() // R++ + if a, ok := p.pool[s]; ok { // found + p.rwm.RUnlock() // R-- + return a + } + + p.rwm.RUnlock() // R-- + // not found but with a race condition, retry within a write lock + p.rwm.Lock() // W++ + defer p.rwm.Unlock() // W-- + if a, ok := p.pool[s]; ok { // done in a race + return a + } + + // we won + s = StrPack(s) + p.pool[s] = s + return s + } + + return +} + +// Count returns the number of items in the pool. +func (p *GoPool) Count() int { + return len(p.pool) +} + +// Dict is a string <-> id bijection. Dict is *not* concurrent access safe for assigning new ids +// to strings not yet contained in the bijection. +// Id for an empty string is guaranteed to be 0, +// thus Id for any non empty string is guaranteed to be non zero. +type Dict struct { + si map[string]int + is []string +} + +// NewDict returns a newly created Dict. +func NewDict() (d *Dict) { + d = &Dict{map[string]int{}, []string{}} + d.Id("") + return +} + +// Count returns the number of items in the dict. +func (d *Dict) Count() int { + return len(d.is) +} + +// Id maps string s to its numeric identificator. +func (d *Dict) Id(s string) (y int) { + if y, ok := d.si[s]; ok { + return y + } + + s = StrPack(s) + y = len(d.is) + d.si[s] = y + d.is = append(d.is, s) + return +} + +// S maps an id to its string value and ok == true. Id values not contained in the bijection +// return "", false. +func (d *Dict) S(id int) (s string, ok bool) { + if id >= len(d.is) { + return "", false + } + return d.is[id], true +} + +// GoDict is a concurrent access safe version of Dict. +type GoDict struct { + si map[string]int + is []string + rwm *sync.RWMutex +} + +// NewGoDict returns a newly created GoDict. +func NewGoDict() (d *GoDict) { + d = &GoDict{map[string]int{}, []string{}, &sync.RWMutex{}} + d.Id("") + return +} + +// Count returns the number of items in the dict. +func (d *GoDict) Count() int { + return len(d.is) +} + +// Id maps string s to its numeric identificator. The implementation honors getting +// an existing id at the cost of assigning a new one. +func (d *GoDict) Id(s string) (y int) { + d.rwm.RLock() // R++ + if y, ok := d.si[s]; ok { // found + d.rwm.RUnlock() // R-- + return y + } + + d.rwm.RUnlock() // R-- + + // not found but with a race condition + d.rwm.Lock() // W++ recheck with write lock + defer d.rwm.Unlock() // W-- + if y, ok := d.si[s]; ok { // some other goroutine won already + return y + } + + // a race free not found state => insert the string + s = StrPack(s) + y = len(d.is) + d.si[s] = y + d.is = append(d.is, s) + return +} + +// S maps an id to its string value and ok == true. Id values not contained in the bijection +// return "", false. +func (d *GoDict) S(id int) (s string, ok bool) { + d.rwm.RLock() // R++ + defer d.rwm.RUnlock() // R-- + if id >= len(d.is) { + return "", false + } + return d.is[id], true +} + +// StrPack returns a new instance of s which is tightly packed in memory. +// It is intended for avoiding the situation where having a live reference +// to a string slice over an unreferenced biger underlying string keeps the biger one +// in memory anyway - it can't be GCed. +func StrPack(s string) string { + return string([]byte(s)) +} + +// JoinFields returns strings in flds joined by sep. Flds may contain arbitrary +// bytes, including the sep as they are safely escaped. JoinFields panics if +// sep is the backslash character or if len(sep) != 1. +func JoinFields(flds []string, sep string) string { + if len(sep) != 1 || sep == "\\" { + panic("invalid separator") + } + + a := make([]string, len(flds)) + for i, v := range flds { + v = strings.Replace(v, "\\", "\\0", -1) + a[i] = strings.Replace(v, sep, "\\1", -1) + } + return strings.Join(a, sep) +} + +// SplitFields splits s, which must be produced by JoinFields using the same +// sep, into flds. SplitFields panics if sep is the backslash character or if +// len(sep) != 1. +func SplitFields(s, sep string) (flds []string) { + if len(sep) != 1 || sep == "\\" { + panic("invalid separator") + } + + a := strings.Split(s, sep) + r := make([]string, len(a)) + for i, v := range a { + v = strings.Replace(v, "\\1", sep, -1) + r[i] = strings.Replace(v, "\\0", "\\", -1) + } + return r +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/AUTHORS b/Godeps/_workspace/src/github.com/cznic/zappy/AUTHORS new file mode 100644 index 00000000000..22c8397d5f8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/AUTHORS @@ -0,0 +1,12 @@ +# This file lists authors for copyright purposes. This file is distinct from +# the CONTRIBUTORS files. See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> +The Snappy-Go Authors diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/CONTRIBUTORS b/Godeps/_workspace/src/github.com/cznic/zappy/CONTRIBUTORS new file mode 100644 index 00000000000..7693bdab74e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/CONTRIBUTORS @@ -0,0 +1,10 @@ +# This file lists people who contributed code to this repository. The AUTHORS +# file lists the copyright holders; this file lists people. +# +# Names should be added to this file like so: +# Name +# +# Please keep the list sorted. + +Jan Mercl <0xjnml@gmail.com> +Mathieu Lonjaret diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/LICENSE b/Godeps/_workspace/src/github.com/cznic/zappy/LICENSE new file mode 100644 index 00000000000..bc67059c529 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 The zappy Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/Makefile b/Godeps/_workspace/src/github.com/cznic/zappy/Makefile new file mode 100644 index 00000000000..c6726d13e06 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/Makefile @@ -0,0 +1,30 @@ +# Copyright 2014 The zappy Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +all: editor + go tool vet -printfuncs Log:0,Logf:0 . + golint . + go install + make todo + +editor: + go fmt + go test -i + @#go test + ./purego.sh + +todo: + @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alnum:]] *.go || true + @grep -n TODO *.go || true + @grep -n BUG *.go || true + @grep -n println *.go || true + +clean: + rm -f *~ cov cov.html + +gocov: + gocov test $(COV) | gocov-html > cov.html + +bench: + go test -run NONE -bench B diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/README.md b/Godeps/_workspace/src/github.com/cznic/zappy/README.md new file mode 100644 index 00000000000..6fc6f68a704 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/README.md @@ -0,0 +1,9 @@ +zappy +===== + +Package zappy implements the zappy block-based compression format. It aims for +a combination of good speed and reasonable compression. + +Installation: $ go get github.com/cznic/zappy + +Documentation: [godoc.org/github.com/cznic/zappy](http://godoc.org/github.com/cznic/zappy) diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/SNAPPY-GO-LICENSE b/Godeps/_workspace/src/github.com/cznic/zappy/SNAPPY-GO-LICENSE new file mode 100644 index 00000000000..6050c10f4c8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/SNAPPY-GO-LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/all_test.go b/Godeps/_workspace/src/github.com/cznic/zappy/all_test.go new file mode 100644 index 00000000000..ed79519d177 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/all_test.go @@ -0,0 +1,379 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +package zappy + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "testing" + + "code.google.com/p/snappy-go/snappy" +) + +var dbg = func(s string, va ...interface{}) { + _, fn, fl, _ := runtime.Caller(1) + fmt.Printf("%s:%d: ", path.Base(fn), fl) + fmt.Printf(s, va...) + fmt.Println() +} + +func use(...interface{}) {} + +var ( + download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") + pureGo = flag.String("purego", "", "verify 'purego' build tag functionality for value `false` or `true`") +) + +func roundtrip(b, ebuf, dbuf []byte) error { + e, err := Encode(ebuf, b) + if err != nil { + return fmt.Errorf("encoding error: %v", err) + } + d, err := Decode(dbuf, e) + if err != nil { + return fmt.Errorf("decoding error: %v", err) + } + if !bytes.Equal(b, d) { + return fmt.Errorf("roundtrip mismatch:\n\twant %v\n\tgot %v", b, d) + } + return nil +} + +func TestEmpty(t *testing.T) { + if err := roundtrip(nil, nil, nil); err != nil { + t.Fatal(err) + } +} + +func TestSmallCopy(t *testing.T) { + for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} { + for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} { + for i := 0; i < 32; i++ { + s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb" + if err := roundtrip([]byte(s), ebuf, dbuf); err != nil { + t.Fatalf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err) + } + } + } + } +} + +func TestSmallRand(t *testing.T) { + rand.Seed(27354294) + for n := 1; n < 20000; n += 23 { + b := make([]byte, n) + for i := range b { + b[i] = uint8(rand.Uint32()) + } + if err := roundtrip(b, nil, nil); err != nil { + t.Fatal(err) + } + } +} + +func TestSmallRegular(t *testing.T) { + for n := 1; n < 20000; n += 23 { + b := make([]byte, n) + for i := range b { + b[i] = uint8(i%10 + 'a') + } + if err := roundtrip(b, nil, nil); err != nil { + t.Fatal(n, err) + } + } +} + +func benchDecode(b *testing.B, src []byte) { + encoded, err := Encode(nil, src) + if err != nil { + b.Fatal(err) + } + // Bandwidth is in amount of uncompressed data. + b.SetBytes(int64(len(src))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Decode(src, encoded) + } +} + +func benchEncode(b *testing.B, src []byte) { + // Bandwidth is in amount of uncompressed data. + b.SetBytes(int64(len(src))) + dst := make([]byte, MaxEncodedLen(len(src))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Encode(dst, src) + } +} + +func readFile(b *testing.B, filename string) []byte { + src, err := ioutil.ReadFile(filename) + if err != nil { + b.Fatalf("failed reading %s: %s", filename, err) + } + if len(src) == 0 { + b.Fatalf("%s has zero length", filename) + } + return src +} + +func readFile2(t *testing.T, filename string) []byte { + src, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatalf("failed reading %s: %s", filename, err) + } + if len(src) == 0 { + t.Fatalf("%s has zero length", filename) + } + return src +} + +// expand returns a slice of length n containing repeated copies of src. +func expand(src []byte, n int) []byte { + dst := make([]byte, n) + for x := dst; len(x) > 0; { + i := copy(x, src) + x = x[i:] + } + return dst +} + +func benchWords(b *testing.B, n int, decode bool) { + // Note: the file is OS-language dependent so the resulting values are not + // directly comparable for non-US-English OS installations. + data := expand(readFile(b, "/usr/share/dict/words"), n) + if decode { + benchDecode(b, data) + } else { + benchEncode(b, data) + } +} + +func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) } +func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) } +func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) } +func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) } +func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) } +func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) } +func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) } +func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) } + +// testFiles' values are copied directly from +// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc. +// The label field is unused in zappy. +var testFiles = []struct { + label string + filename string +}{ + {"html", "html"}, + {"urls", "urls.10K"}, + {"jpg", "house.jpg"}, + {"pdf", "mapreduce-osdi-1.pdf"}, + {"html4", "html_x_4"}, + {"cp", "cp.html"}, + {"c", "fields.c"}, + {"lsp", "grammar.lsp"}, + {"xls", "kennedy.xls"}, + {"txt1", "alice29.txt"}, + {"txt2", "asyoulik.txt"}, + {"txt3", "lcet10.txt"}, + {"txt4", "plrabn12.txt"}, + {"bin", "ptt5"}, + {"sum", "sum"}, + {"man", "xargs.1"}, + {"pb", "geo.protodata"}, + {"gaviota", "kppkn.gtb"}, +} + +// The test data files are present at this canonical URL. +const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/" + +func downloadTestdata(basename string) (errRet error) { + filename := filepath.Join("testdata", basename) + f, err := os.Create(filename) + if err != nil { + return fmt.Errorf("failed to create %s: %s", filename, err) + } + + defer f.Close() + defer func() { + if errRet != nil { + os.Remove(filename) + } + }() + resp, err := http.Get(baseURL + basename) + if err != nil { + return fmt.Errorf("failed to download %s: %s", baseURL+basename, err) + } + defer resp.Body.Close() + _, err = io.Copy(f, resp.Body) + if err != nil { + return fmt.Errorf("failed to write %s: %s", filename, err) + } + return nil +} + +func benchFile(b *testing.B, n int, decode bool) { + filename := filepath.Join("testdata", testFiles[n].filename) + if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 { + if !*download { + b.Fatal("test data not found; skipping benchmark without the -download flag") + } + // Download the official snappy C++ implementation reference test data + // files for benchmarking. + if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) { + b.Fatalf("failed to create testdata: %s", err) + } + for _, tf := range testFiles { + if err := downloadTestdata(tf.filename); err != nil { + b.Fatalf("failed to download testdata: %s", err) + } + } + } + data := readFile(b, filename) + if decode { + benchDecode(b, data) + } else { + benchEncode(b, data) + } +} + +// Naming convention is kept similar to what snappy's C++ implementation uses. +func Benchmark_UFlat0(b *testing.B) { benchFile(b, 0, true) } +func Benchmark_UFlat1(b *testing.B) { benchFile(b, 1, true) } +func Benchmark_UFlat2(b *testing.B) { benchFile(b, 2, true) } +func Benchmark_UFlat3(b *testing.B) { benchFile(b, 3, true) } +func Benchmark_UFlat4(b *testing.B) { benchFile(b, 4, true) } +func Benchmark_UFlat5(b *testing.B) { benchFile(b, 5, true) } +func Benchmark_UFlat6(b *testing.B) { benchFile(b, 6, true) } +func Benchmark_UFlat7(b *testing.B) { benchFile(b, 7, true) } +func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) } +func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) } +func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) } +func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) } +func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) } +func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) } +func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) } +func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) } +func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) } +func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) } +func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) } +func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) } +func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) } +func Benchmark_ZFlat3(b *testing.B) { benchFile(b, 3, false) } +func Benchmark_ZFlat4(b *testing.B) { benchFile(b, 4, false) } +func Benchmark_ZFlat5(b *testing.B) { benchFile(b, 5, false) } +func Benchmark_ZFlat6(b *testing.B) { benchFile(b, 6, false) } +func Benchmark_ZFlat7(b *testing.B) { benchFile(b, 7, false) } +func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) } +func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) } +func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) } +func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) } +func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) } +func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) } +func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) } +func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) } +func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) } +func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) } + +func TestCmp(t *testing.T) { + var ts, tz, to int + for i := 0; i <= 17; i++ { + filename := filepath.Join("testdata", testFiles[i].filename) + if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 { + if !*download { + t.Fatal("test data not found; skipping test without the -download flag") + } + // Download the official snappy C++ implementation reference test data + // files for benchmarking. + if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) { + t.Fatalf("failed to create testdata: %s", err) + } + for _, tf := range testFiles { + if err := downloadTestdata(tf.filename); err != nil { + t.Fatalf("failed to download testdata: %s", err) + } + } + } + data := readFile2(t, filename) + orig := len(data) + to += orig + senc, err := snappy.Encode(nil, data) + if err != nil { + t.Fatal(err) + } + + ns := len(senc) + zenc, err := Encode(nil, data) + if err != nil { + t.Fatal(err) + } + + nz := len(zenc) + t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", filename, ns, nz, float64(nz)/float64(ns), orig) + ts += ns + tz += nz + } + t.Logf("%35s: snappy %7d, zappy %7d, %.3f, orig %7d", "TOTAL", ts, tz, float64(tz)/float64(ts), to) +} + +func TestBitIndex(t *testing.T) { + rng := rand.New(rand.NewSource(42)) + for n := 16; n <= 1<<16; n <<= 1 { + data := make([]byte, n) + for i := 0; i < n/1000+1; i++ { + data[rng.Int()%n] = 1 + } + + senc, err := snappy.Encode(nil, data) + if err != nil { + t.Fatal(err) + } + + ns := len(senc) + zenc, err := Encode(nil, data) + if err != nil { + t.Fatal(err) + } + + nz := len(zenc) + t.Logf("Sparse bit index %7d B: snappy %7d, zappy %7d, %.3f", n, ns, nz, float64(nz)/float64(ns)) + } +} + +func TestPureGo(t *testing.T) { + var purego bool + switch s := *pureGo; s { + case "false": + // nop + case "true": + purego = true + default: + t.Logf("Not performed: %q", s) + return + } + + if g, e := puregoDecode(), purego; g != e { + t.Fatal("Decode", g, e) + } + + if g, e := puregoEncode(), purego; g != e { + t.Fatal("Encode", g, e) + } +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/decode.go b/Godeps/_workspace/src/github.com/cznic/zappy/decode.go new file mode 100644 index 00000000000..867c8deea5f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/decode.go @@ -0,0 +1,38 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +package zappy + +import ( + "encoding/binary" + "errors" +) + +// ErrCorrupt reports that the input is invalid. +var ErrCorrupt = errors.New("zappy: corrupt input") + +// DecodedLen returns the length of the decoded block. +func DecodedLen(src []byte) (int, error) { + v, _, err := decodedLen(src) + return v, err +} + +// decodedLen returns the length of the decoded block and the number of bytes +// that the length header occupied. +func decodedLen(src []byte) (blockLen, headerLen int, err error) { + v, n := binary.Uvarint(src) + if n == 0 { + return 0, 0, ErrCorrupt + } + + if uint64(int(v)) != v { + return 0, 0, errors.New("zappy: decoded block is too large") + } + + return int(v), n, nil +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/decode_cgo.go b/Godeps/_workspace/src/github.com/cznic/zappy/decode_cgo.go new file mode 100644 index 00000000000..993c99242b4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/decode_cgo.go @@ -0,0 +1,121 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +// +build cgo,!purego + +package zappy + +/* + +#include +#include + +// supports only uint32 encoded values +int uvarint(unsigned int* n, uint8_t* src, int len) { + int r = 0; + unsigned int v = 0; + unsigned int s = 0; + while ((len-- != 0) && (++r <= 5)) { + uint8_t b = *src++; + v = v | ((b&0x7f)<>1; + if ((u&1) != 0) + x = ~x; + *n = x; + return i; +} + +int decode(int s, int len_src, uint8_t* src, int len_dst, uint8_t* dst) { + int d = 0; + int length; + while (s < len_src) { + int n, i = varint(&n, src+s, len_src-s); + if (i <= 0) { + return -1; + } + + s += i; + if (n >= 0) { + length = n+1; + if ((length > len_dst-d) || (length > len_src-s)) + return -1; + + memcpy(dst+d, src+s, length); + d += length; + s += length; + continue; + } + + + length = -n; + int offset; + i = uvarint((unsigned int*)(&offset), src+s, len_src-s); + if (i <= 0) + return -1; + + s += i; + if (s > len_src) + return -1; + + int end = d+length; + if ((offset > d) || (end > len_dst)) + return -1; + + for( ; d < end; d++) + *(dst+d) = *(dst+d-offset); + } + return d; +} + +*/ +import "C" + +func puregoDecode() bool { return false } + +// Decode returns the decoded form of src. The returned slice may be a sub- +// slice of buf if buf was large enough to hold the entire decoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil buf. +func Decode(buf, src []byte) ([]byte, error) { + dLen, s, err := decodedLen(src) + if err != nil { + return nil, err + } + + if dLen == 0 { + if len(src) == 1 { + return nil, nil + } + + return nil, ErrCorrupt + } + + if len(buf) < dLen { + buf = make([]byte, dLen) + } + + d := int(C.decode(C.int(s), C.int(len(src)), (*C.uint8_t)(&src[0]), C.int(len(buf)), (*C.uint8_t)(&buf[0]))) + if d != dLen { + return nil, ErrCorrupt + } + + return buf[:d], nil +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/decode_nocgo.go b/Godeps/_workspace/src/github.com/cznic/zappy/decode_nocgo.go new file mode 100644 index 00000000000..75b782f8fae --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/decode_nocgo.go @@ -0,0 +1,89 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +// +build !cgo purego + +package zappy + +import ( + "encoding/binary" +) + +func puregoDecode() bool { return true } + +// Decode returns the decoded form of src. The returned slice may be a sub- +// slice of buf if buf was large enough to hold the entire decoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil buf. +func Decode(buf, src []byte) ([]byte, error) { + dLen, s, err := decodedLen(src) + if err != nil { + return nil, err + } + + if dLen == 0 { + if len(src) == 1 { + return nil, nil + } + + return nil, ErrCorrupt + } + + if len(buf) < dLen { + buf = make([]byte, dLen) + } + + var d, offset, length int + for s < len(src) { + n, i := binary.Varint(src[s:]) + if i <= 0 { + return nil, ErrCorrupt + } + + s += i + if n >= 0 { + length = int(n + 1) + if length > len(buf)-d || length > len(src)-s { + return nil, ErrCorrupt + } + + copy(buf[d:], src[s:s+length]) + d += length + s += length + continue + } + + length = int(-n) + off64, i := binary.Uvarint(src[s:]) + if i <= 0 { + return nil, ErrCorrupt + } + + offset = int(off64) + s += i + if s > len(src) { + return nil, ErrCorrupt + } + + end := d + length + if offset > d || end > len(buf) { + return nil, ErrCorrupt + } + + for s, v := range buf[d-offset : end-offset] { + buf[d+s] = v + } + d = end + + } + if d != dLen { + return nil, ErrCorrupt + } + + return buf[:d], nil +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/encode.go b/Godeps/_workspace/src/github.com/cznic/zappy/encode.go new file mode 100644 index 00000000000..27ceba03b87 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/encode.go @@ -0,0 +1,37 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +package zappy + +import ( + "encoding/binary" +) + +// We limit how far copy back-references can go, the same as the snappy C++ +// code. +const maxOffset = 1 << 20 + +// emitCopy writes a copy chunk and returns the number of bytes written. +func emitCopy(dst []byte, offset, length int) (n int) { + n = binary.PutVarint(dst, int64(-length)) + n += binary.PutUvarint(dst[n:], uint64(offset)) + return +} + +// emitLiteral writes a literal chunk and returns the number of bytes written. +func emitLiteral(dst, lit []byte) (n int) { + n = binary.PutVarint(dst, int64(len(lit)-1)) + n += copy(dst[n:], lit) + return +} + +// MaxEncodedLen returns the maximum length of a zappy block, given its +// uncompressed length. +func MaxEncodedLen(srcLen int) int { + return 10 + srcLen +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/encode_cgo.go b/Godeps/_workspace/src/github.com/cznic/zappy/encode_cgo.go new file mode 100644 index 00000000000..6c343f58eb2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/encode_cgo.go @@ -0,0 +1,140 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +// +build cgo,!purego + +package zappy + +/* + +#include +#include + +#define MAXOFFSET 1<<20 + +int putUvarint(uint8_t* buf, unsigned int x) { + int i = 1; + for (; x >= 0x80; i++) { + *buf++ = x|0x80; + x >>= 7; + } + *buf = x; + return i; +} + +int putVarint(uint8_t* buf, int x) { + unsigned int ux = x << 1; + if (x < 0) + ux = ~ux; + return putUvarint(buf, ux); +} + +int emitLiteral(uint8_t* dst, uint8_t* lit, int len_lit) { + int n = putVarint(dst, len_lit-1); + memcpy(dst+n, lit, len_lit); + return n+len_lit; +} + +int emitCopy(uint8_t* dst, int off, int len) { + int n = putVarint(dst, -len); + return n+putUvarint(dst+n, (unsigned int)off); +} + +int encode(int d, uint8_t* dst, uint8_t* src, int len_src) { + int table[1<<12]; + int s = 0; + int t = 0; + int lit = 0; + int lim = 0; + memset(table, 0, sizeof(table)); + for (lim = len_src-3; s < lim; ) { + // Update the hash table. + uint32_t b0 = src[s]; + uint32_t b1 = src[s+1]; + uint32_t b2 = src[s+2]; + uint32_t b3 = src[s+3]; + uint32_t h = b0 | (b1<<8) | (b2<<16) | (b3<<24); + uint32_t i; +more: + i = (h*0x1e35a7bd)>>20; + t = table[i]; + table[i] = s; + // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. + if ((t == 0) || (s-t >= MAXOFFSET) || (b0 != src[t]) || (b1 != src[t+1]) || (b2 != src[t+2]) || (b3 != src[t+3])) { + s++; + if (s >= lim) + break; + + b0 = b1; + b1 = b2; + b2 = b3; + b3 = src[s+3]; + h = (h>>8) | ((b3)<<24); + goto more; + } + + // Otherwise, we have a match. First, emit any pending literal bytes. + if (lit != s) { + d += emitLiteral(dst+d, src+lit, s-lit); + } + // Extend the match to be as long as possible. + int s0 = s; + s += 4; + t += 4; + while ((s < len_src) && (src[s] == src[t])) { + s++; + t++; + } + d += emitCopy(dst+d, s-t, s-s0); + lit = s; + } + // Emit any final pending literal bytes and return. + if (lit != len_src) { + d += emitLiteral(dst+d, src+lit, len_src-lit); + } + return d; +} + +*/ +import "C" + +import ( + "encoding/binary" + "fmt" + "math" +) + +func puregoEncode() bool { return false } + +// Encode returns the encoded form of src. The returned slice may be a sub- +// slice of buf if buf was large enough to hold the entire encoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil buf. +func Encode(buf, src []byte) ([]byte, error) { + if n := MaxEncodedLen(len(src)); len(buf) < n { + buf = make([]byte, n) + } + + if len(src) > math.MaxInt32 { + return nil, fmt.Errorf("zappy.Encode: too long data: %d bytes", len(src)) + } + + // The block starts with the varint-encoded length of the decompressed bytes. + d := binary.PutUvarint(buf, uint64(len(src))) + + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + d += emitLiteral(buf[d:], src) + } + return buf[:d], nil + } + + d = int(C.encode(C.int(d), (*C.uint8_t)(&buf[0]), (*C.uint8_t)(&src[0]), C.int(len(src)))) + return buf[:d], nil +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/encode_nocgo.go b/Godeps/_workspace/src/github.com/cznic/zappy/encode_nocgo.go new file mode 100644 index 00000000000..1d1df44395e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/encode_nocgo.go @@ -0,0 +1,92 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +// +build !cgo purego + +package zappy + +import ( + "encoding/binary" + "fmt" + "math" +) + +func puregoEncode() bool { return true } + +// Encode returns the encoded form of src. The returned slice may be a sub- +// slice of buf if buf was large enough to hold the entire encoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil buf. +func Encode(buf, src []byte) ([]byte, error) { + if n := MaxEncodedLen(len(src)); len(buf) < n { + buf = make([]byte, n) + } + + if len(src) > math.MaxInt32 { + return nil, fmt.Errorf("zappy.Encode: too long data: %d bytes", len(src)) + } + + // The block starts with the varint-encoded length of the decompressed bytes. + d := binary.PutUvarint(buf, uint64(len(src))) + + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + d += emitLiteral(buf[d:], src) + } + return buf[:d], nil + } + + // Iterate over the source bytes. + var ( + table [1 << 12]int // Hash table + s int // The iterator position. + t int // The last position with the same hash as s. + lit int // The start position of any pending literal bytes. + ) + for lim := len(src) - 3; s < lim; { + // Update the hash table. + b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3] + h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24 + more: + p := &table[(h*0x1e35a7bd)>>20] + t, *p = *p, s + // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. + if t == 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] { + s++ + if s >= lim { + break + } + + b0, b1, b2, b3 = b1, b2, b3, src[s+3] + h = h>>8 | uint32(b3)<<24 + goto more + } + + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + d += emitLiteral(buf[d:], src[lit:s]) + } + // Extend the match to be as long as possible. + s0 := s + s, t = s+4, t+4 + for s < len(src) && src[s] == src[t] { + s++ + t++ + } + // Emit the copied bytes. + d += emitCopy(buf[d:], s-t, s-s0) + lit = s + } + + // Emit any final pending literal bytes and return. + if lit != len(src) { + d += emitLiteral(buf[d:], src[lit:]) + } + return buf[:d], nil +} diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/purego.sh b/Godeps/_workspace/src/github.com/cznic/zappy/purego.sh new file mode 100644 index 00000000000..e2fec14e63e --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/purego.sh @@ -0,0 +1,11 @@ +# Copyright 2014 The zappy Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e +set -v + +CGO_ENABLED=0 go test -purego true +CGO_ENABLED=0 go test -purego true -tags purego +CGO_ENABLED=1 go test -purego false +CGO_ENABLED=1 go test -purego true -tags purego diff --git a/Godeps/_workspace/src/github.com/cznic/zappy/zappy.go b/Godeps/_workspace/src/github.com/cznic/zappy/zappy.go new file mode 100644 index 00000000000..760df34a150 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cznic/zappy/zappy.go @@ -0,0 +1,241 @@ +// Copyright 2014 The zappy Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the SNAPPY-GO-LICENSE file. + +/* +Package zappy implements the zappy block-based compression format. It aims for +a combination of good speed and reasonable compression. + +Zappy is a format incompatible, API compatible fork of snappy-go[1]. The C++ +snappy implementation is at [2]. + +Reasons for the fork + +The snappy compression is pretty good. Yet it has one problem built into its +format definition[3] - the maximum length of a copy "instruction" is 64 bytes. +For some specific usage patterns with long runs of repeated data, it turns out +the compression is suboptimal. For example a 1:1000 "sparseness" 64kB bit index +with only few set bits is compressed to about 3kB (about 1000 of 64B copy, 3 +byte "instructions"). + +Format description + +Zappy uses much less complicated format than snappy. Each encoded block begins +with the uvarint-encoded[4] length of the decoded data, followed by a sequence +of chunks. Chunks begin and end on byte boundaries. The chunk starts with a +varint encoded number N: + + N >= 0: N+1 literal bytes follow. + + N < 0: copy -N bytes, starting at offset M (in the following uvarint). + +Performance issues + +Compression rate is roughly the same as of snappy for the reference data set: + + testdata/html: snappy 23320, zappy 22943, 0.984, orig 102400 + testdata/urls.10K: snappy 334437, zappy 355163, 1.062, orig 702087 + testdata/house.jpg: snappy 126711, zappy 126694, 1.000, orig 126958 + testdata/mapreduce-osdi-1.pdf: snappy 77227, zappy 77646, 1.005, orig 94330 + testdata/html_x_4: snappy 92350, zappy 22956, 0.249, orig 409600 + testdata/cp.html: snappy 11938, zappy 12961, 1.086, orig 24603 + testdata/fields.c: snappy 4825, zappy 5395, 1.118, orig 11150 + testdata/grammar.lsp: snappy 1814, zappy 1933, 1.066, orig 3721 + testdata/kennedy.xls: snappy 423518, zappy 440597, 1.040, orig 1029744 + testdata/alice29.txt: snappy 89550, zappy 104016, 1.162, orig 152089 + testdata/asyoulik.txt: snappy 79583, zappy 91345, 1.148, orig 125179 + testdata/lcet10.txt: snappy 238761, zappy 275488, 1.154, orig 426754 + testdata/plrabn12.txt: snappy 324567, zappy 376885, 1.161, orig 481861 + testdata/ptt5: snappy 96350, zappy 91465, 0.949, orig 513216 + testdata/sum: snappy 18927, zappy 20015, 1.057, orig 38240 + testdata/xargs.1: snappy 2532, zappy 2793, 1.103, orig 4227 + testdata/geo.protodata: snappy 23362, zappy 20759, 0.889, orig 118588 + testdata/kppkn.gtb: snappy 73962, zappy 87200, 1.179, orig 184320 + TOTAL: snappy 2043734, zappy 2136254, 1.045, orig 4549067 + +Zappy has better RLE handling (1/1000+1 non zero bytes in each index): + + Sparse bit index 16 B: snappy 9, zappy 9, 1.000 + Sparse bit index 32 B: snappy 10, zappy 10, 1.000 + Sparse bit index 64 B: snappy 11, zappy 10, 0.909 + Sparse bit index 128 B: snappy 16, zappy 14, 0.875 + Sparse bit index 256 B: snappy 22, zappy 14, 0.636 + Sparse bit index 512 B: snappy 36, zappy 16, 0.444 + Sparse bit index 1024 B: snappy 57, zappy 18, 0.316 + Sparse bit index 2048 B: snappy 111, zappy 32, 0.288 + Sparse bit index 4096 B: snappy 210, zappy 31, 0.148 + Sparse bit index 8192 B: snappy 419, zappy 75, 0.179 + Sparse bit index 16384 B: snappy 821, zappy 138, 0.168 + Sparse bit index 32768 B: snappy 1627, zappy 232, 0.143 + Sparse bit index 65536 B: snappy 3243, zappy 451, 0.139 + +When compiled with CGO_ENABLED=1, zappy is now faster than snappy-go. +Old=snappy-go, new=zappy: + + benchmark old MB/s new MB/s speedup + BenchmarkWordsDecode1e3 148.98 189.04 1.27x + BenchmarkWordsDecode1e4 150.29 182.51 1.21x + BenchmarkWordsDecode1e5 145.79 182.95 1.25x + BenchmarkWordsDecode1e6 167.43 187.69 1.12x + BenchmarkWordsEncode1e3 47.11 145.69 3.09x + BenchmarkWordsEncode1e4 81.47 136.50 1.68x + BenchmarkWordsEncode1e5 78.86 127.93 1.62x + BenchmarkWordsEncode1e6 96.81 142.95 1.48x + Benchmark_UFlat0 316.87 463.19 1.46x + Benchmark_UFlat1 231.56 350.32 1.51x + Benchmark_UFlat2 3656.68 8258.39 2.26x + Benchmark_UFlat3 892.56 1270.09 1.42x + Benchmark_UFlat4 315.84 959.08 3.04x + Benchmark_UFlat5 211.70 301.55 1.42x + Benchmark_UFlat6 211.59 258.29 1.22x + Benchmark_UFlat7 209.80 272.21 1.30x + Benchmark_UFlat8 254.59 301.70 1.19x + Benchmark_UFlat9 163.39 192.66 1.18x + Benchmark_UFlat10 155.46 189.70 1.22x + Benchmark_UFlat11 170.11 198.95 1.17x + Benchmark_UFlat12 148.32 178.78 1.21x + Benchmark_UFlat13 359.25 579.99 1.61x + Benchmark_UFlat14 197.27 291.33 1.48x + Benchmark_UFlat15 185.75 248.07 1.34x + Benchmark_UFlat16 362.74 582.66 1.61x + Benchmark_UFlat17 222.95 240.01 1.08x + Benchmark_ZFlat0 188.66 311.89 1.65x + Benchmark_ZFlat1 101.46 201.34 1.98x + Benchmark_ZFlat2 93.62 244.50 2.61x + Benchmark_ZFlat3 102.79 243.34 2.37x + Benchmark_ZFlat4 191.64 625.32 3.26x + Benchmark_ZFlat5 103.09 169.39 1.64x + Benchmark_ZFlat6 110.35 182.57 1.65x + Benchmark_ZFlat7 89.56 190.53 2.13x + Benchmark_ZFlat8 154.05 235.68 1.53x + Benchmark_ZFlat9 87.58 133.51 1.52x + Benchmark_ZFlat10 82.08 127.51 1.55x + Benchmark_ZFlat11 91.36 138.91 1.52x + Benchmark_ZFlat12 79.24 123.02 1.55x + Benchmark_ZFlat13 217.04 374.26 1.72x + Benchmark_ZFlat14 100.33 168.03 1.67x + Benchmark_ZFlat15 80.79 160.46 1.99x + Benchmark_ZFlat16 213.32 375.79 1.76x + Benchmark_ZFlat17 135.37 197.13 1.46x + +The package builds with CGO_ENABLED=0 as well, but the performance is worse. + + + $ CGO_ENABLED=0 go test -test.run=NONE -test.bench=. > old.benchcmp + $ CGO_ENABLED=1 go test -test.run=NONE -test.bench=. > new.benchcmp + $ benchcmp old.benchcmp new.benchcmp + benchmark old ns/op new ns/op delta + BenchmarkWordsDecode1e3 9735 5288 -45.68% + BenchmarkWordsDecode1e4 100229 55369 -44.76% + BenchmarkWordsDecode1e5 1037611 546420 -47.34% + BenchmarkWordsDecode1e6 9559352 5335307 -44.19% + BenchmarkWordsEncode1e3 16206 6629 -59.10% + BenchmarkWordsEncode1e4 140283 73161 -47.85% + BenchmarkWordsEncode1e5 1476657 781756 -47.06% + BenchmarkWordsEncode1e6 12702229 6997656 -44.91% + Benchmark_UFlat0 397307 221198 -44.33% + Benchmark_UFlat1 3890483 2008341 -48.38% + Benchmark_UFlat2 35810 15398 -57.00% + Benchmark_UFlat3 140850 74194 -47.32% + Benchmark_UFlat4 814575 426783 -47.61% + Benchmark_UFlat5 156995 81473 -48.10% + Benchmark_UFlat6 77645 43161 -44.41% + Benchmark_UFlat7 25415 13579 -46.57% + Benchmark_UFlat8 6372440 3412916 -46.44% + Benchmark_UFlat9 1453679 789956 -45.66% + Benchmark_UFlat10 1243146 660747 -46.85% + Benchmark_UFlat11 3903493 2146334 -45.02% + Benchmark_UFlat12 5106250 2696144 -47.20% + Benchmark_UFlat13 1641394 884969 -46.08% + Benchmark_UFlat14 262206 131174 -49.97% + Benchmark_UFlat15 32325 17047 -47.26% + Benchmark_UFlat16 366991 204877 -44.17% + Benchmark_UFlat17 1343988 770907 -42.64% + Benchmark_ZFlat0 579954 329812 -43.13% + Benchmark_ZFlat1 6564692 3504867 -46.61% + Benchmark_ZFlat2 902029 513700 -43.05% + Benchmark_ZFlat3 678722 384312 -43.38% + Benchmark_ZFlat4 1197389 654361 -45.35% + Benchmark_ZFlat5 262677 144939 -44.82% + Benchmark_ZFlat6 111249 60876 -45.28% + Benchmark_ZFlat7 39024 19420 -50.24% + Benchmark_ZFlat8 8046106 4387928 -45.47% + Benchmark_ZFlat9 2043167 1143139 -44.05% + Benchmark_ZFlat10 1781604 980528 -44.96% + Benchmark_ZFlat11 5478647 3078585 -43.81% + Benchmark_ZFlat12 7245995 3929863 -45.77% + Benchmark_ZFlat13 2432529 1371606 -43.61% + Benchmark_ZFlat14 420315 227494 -45.88% + Benchmark_ZFlat15 52378 26564 -49.28% + Benchmark_ZFlat16 567047 316196 -44.24% + Benchmark_ZFlat17 1630820 937310 -42.53% + + benchmark old MB/s new MB/s speedup + BenchmarkWordsDecode1e3 102.71 189.08 1.84x + BenchmarkWordsDecode1e4 99.77 180.60 1.81x + BenchmarkWordsDecode1e5 96.38 183.01 1.90x + BenchmarkWordsDecode1e6 104.61 187.43 1.79x + BenchmarkWordsEncode1e3 61.70 150.85 2.44x + BenchmarkWordsEncode1e4 71.28 136.68 1.92x + BenchmarkWordsEncode1e5 67.72 127.92 1.89x + BenchmarkWordsEncode1e6 78.73 142.90 1.82x + Benchmark_UFlat0 257.73 462.93 1.80x + Benchmark_UFlat1 180.46 349.59 1.94x + Benchmark_UFlat2 3545.30 8244.61 2.33x + Benchmark_UFlat3 669.72 1271.39 1.90x + Benchmark_UFlat4 502.84 959.74 1.91x + Benchmark_UFlat5 156.71 301.98 1.93x + Benchmark_UFlat6 143.60 258.33 1.80x + Benchmark_UFlat7 146.41 274.01 1.87x + Benchmark_UFlat8 161.59 301.72 1.87x + Benchmark_UFlat9 104.62 192.53 1.84x + Benchmark_UFlat10 100.70 189.45 1.88x + Benchmark_UFlat11 109.33 198.83 1.82x + Benchmark_UFlat12 94.37 178.72 1.89x + Benchmark_UFlat13 312.67 579.93 1.85x + Benchmark_UFlat14 145.84 291.52 2.00x + Benchmark_UFlat15 130.77 247.95 1.90x + Benchmark_UFlat16 323.14 578.82 1.79x + Benchmark_UFlat17 137.14 239.09 1.74x + Benchmark_ZFlat0 176.57 310.48 1.76x + Benchmark_ZFlat1 106.95 200.32 1.87x + Benchmark_ZFlat2 140.75 247.14 1.76x + Benchmark_ZFlat3 138.98 245.45 1.77x + Benchmark_ZFlat4 342.08 625.95 1.83x + Benchmark_ZFlat5 93.66 169.75 1.81x + Benchmark_ZFlat6 100.23 183.16 1.83x + Benchmark_ZFlat7 95.35 191.60 2.01x + Benchmark_ZFlat8 127.98 234.68 1.83x + Benchmark_ZFlat9 74.44 133.04 1.79x + Benchmark_ZFlat10 70.26 127.66 1.82x + Benchmark_ZFlat11 77.89 138.62 1.78x + Benchmark_ZFlat12 66.50 122.62 1.84x + Benchmark_ZFlat13 210.98 374.17 1.77x + Benchmark_ZFlat14 90.98 168.09 1.85x + Benchmark_ZFlat15 80.70 159.12 1.97x + Benchmark_ZFlat16 209.13 375.04 1.79x + Benchmark_ZFlat17 113.02 196.65 1.74x + $ + +Build tags + +If a constraint 'purego' appears in the build constraints [5] then a pure Go +version is built regardless of the $CGO_ENABLED value. + + $ touch zappy.go ; go install -tags purego github.com/cznic/zappy # for example + +Information sources + +... referenced from the above documentation. + + [1]: http://code.google.com/p/snappy-go/ + [2]: http://code.google.com/p/snappy/ + [3]: http://code.google.com/p/snappy/source/browse/trunk/format_description.txt + [4]: http://golang.org/pkg/encoding/binary/ + [5]: http://golang.org/pkg/go/build/#hdr-Build_Constraints +*/ +package zappy diff --git a/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/avgvar.go b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/avgvar.go new file mode 100644 index 00000000000..2d7e2a3262d --- /dev/null +++ b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/avgvar.go @@ -0,0 +1,39 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +import "math" + +// avgVar maintains the average and variance of a stream of numbers +// in a space-efficient manner. +type avgVar struct { + count int64 + sum, sumsq float64 +} + +func (av *avgVar) Init() { + av.count = 0 + av.sum = 0.0 + av.sumsq = 0.0 +} + +func (av *avgVar) Add(sample float64) { + av.count++ + av.sum += sample + av.sumsq += sample * sample +} + +func (av *avgVar) GetCount() int64 { return av.count } + +func (av *avgVar) GetAvg() float64 { return av.sum / float64(av.count) } + +func (av *avgVar) GetTotal() float64 { return av.sum } + +func (av *avgVar) GetVar() float64 { + a := av.GetAvg() + return av.sumsq/float64(av.count) - a*a +} + +func (av *avgVar) GetStdDev() float64 { return math.Sqrt(av.GetVar()) } diff --git a/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/iterator.go b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/iterator.go new file mode 100644 index 00000000000..ee7b27f442b --- /dev/null +++ b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/iterator.go @@ -0,0 +1,93 @@ +package llrb + +type ItemIterator func(i Item) bool + +//func (t *Tree) Ascend(iterator ItemIterator) { +// t.AscendGreaterOrEqual(Inf(-1), iterator) +//} + +func (t *LLRB) AscendRange(greaterOrEqual, lessThan Item, iterator ItemIterator) { + t.ascendRange(t.root, greaterOrEqual, lessThan, iterator) +} + +func (t *LLRB) ascendRange(h *Node, inf, sup Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if !less(h.Item, sup) { + return t.ascendRange(h.Left, inf, sup, iterator) + } + if less(h.Item, inf) { + return t.ascendRange(h.Right, inf, sup, iterator) + } + + if !t.ascendRange(h.Left, inf, sup, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + return t.ascendRange(h.Right, inf, sup, iterator) +} + +// AscendGreaterOrEqual will call iterator once for each element greater or equal to +// pivot in ascending order. It will stop whenever the iterator returns false. +func (t *LLRB) AscendGreaterOrEqual(pivot Item, iterator ItemIterator) { + t.ascendGreaterOrEqual(t.root, pivot, iterator) +} + +func (t *LLRB) ascendGreaterOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if !less(h.Item, pivot) { + if !t.ascendGreaterOrEqual(h.Left, pivot, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + } + return t.ascendGreaterOrEqual(h.Right, pivot, iterator) +} + +func (t *LLRB) AscendLessThan(pivot Item, iterator ItemIterator) { + t.ascendLessThan(t.root, pivot, iterator) +} + +func (t *LLRB) ascendLessThan(h *Node, pivot Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if !t.ascendLessThan(h.Left, pivot, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + if less(h.Item, pivot) { + return t.ascendLessThan(h.Left, pivot, iterator) + } + return true +} + +// DescendLessOrEqual will call iterator once for each element less than the +// pivot in descending order. It will stop whenever the iterator returns false. +func (t *LLRB) DescendLessOrEqual(pivot Item, iterator ItemIterator) { + t.descendLessOrEqual(t.root, pivot, iterator) +} + +func (t *LLRB) descendLessOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if less(h.Item, pivot) || !less(pivot, h.Item) { + if !t.descendLessOrEqual(h.Right, pivot, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + } + return t.descendLessOrEqual(h.Left, pivot, iterator) +} diff --git a/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/iterator_test.go b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/iterator_test.go new file mode 100644 index 00000000000..db5e12c92eb --- /dev/null +++ b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/iterator_test.go @@ -0,0 +1,76 @@ +package llrb + +import ( + "reflect" + "testing" +) + +func TestAscendGreaterOrEqual(t *testing.T) { + tree := New() + tree.InsertNoReplace(Int(4)) + tree.InsertNoReplace(Int(6)) + tree.InsertNoReplace(Int(1)) + tree.InsertNoReplace(Int(3)) + var ary []Item + tree.AscendGreaterOrEqual(Int(-1), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected := []Item{Int(1), Int(3), Int(4), Int(6)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.AscendGreaterOrEqual(Int(3), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(3), Int(4), Int(6)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.AscendGreaterOrEqual(Int(2), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(3), Int(4), Int(6)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } +} + +func TestDescendLessOrEqual(t *testing.T) { + tree := New() + tree.InsertNoReplace(Int(4)) + tree.InsertNoReplace(Int(6)) + tree.InsertNoReplace(Int(1)) + tree.InsertNoReplace(Int(3)) + var ary []Item + tree.DescendLessOrEqual(Int(10), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected := []Item{Int(6), Int(4), Int(3), Int(1)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.DescendLessOrEqual(Int(4), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(4), Int(3), Int(1)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.DescendLessOrEqual(Int(5), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(4), Int(3), Int(1)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } +} diff --git a/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb-stats.go b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb-stats.go new file mode 100644 index 00000000000..47126a3be96 --- /dev/null +++ b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb-stats.go @@ -0,0 +1,46 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +// GetHeight() returns an item in the tree with key @key, and it's height in the tree +func (t *LLRB) GetHeight(key Item) (result Item, depth int) { + return t.getHeight(t.root, key) +} + +func (t *LLRB) getHeight(h *Node, item Item) (Item, int) { + if h == nil { + return nil, 0 + } + if less(item, h.Item) { + result, depth := t.getHeight(h.Left, item) + return result, depth + 1 + } + if less(h.Item, item) { + result, depth := t.getHeight(h.Right, item) + return result, depth + 1 + } + return h.Item, 0 +} + +// HeightStats() returns the average and standard deviation of the height +// of elements in the tree +func (t *LLRB) HeightStats() (avg, stddev float64) { + av := &avgVar{} + heightStats(t.root, 0, av) + return av.GetAvg(), av.GetStdDev() +} + +func heightStats(h *Node, d int, av *avgVar) { + if h == nil { + return + } + av.Add(float64(d)) + if h.Left != nil { + heightStats(h.Left, d+1, av) + } + if h.Right != nil { + heightStats(h.Right, d+1, av) + } +} diff --git a/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb.go b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb.go new file mode 100644 index 00000000000..81373fbfdf0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb.go @@ -0,0 +1,456 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A Left-Leaning Red-Black (LLRB) implementation of 2-3 balanced binary search trees, +// based on the following work: +// +// http://www.cs.princeton.edu/~rs/talks/LLRB/08Penn.pdf +// http://www.cs.princeton.edu/~rs/talks/LLRB/LLRB.pdf +// http://www.cs.princeton.edu/~rs/talks/LLRB/Java/RedBlackBST.java +// +// 2-3 trees (and the run-time equivalent 2-3-4 trees) are the de facto standard BST +// algoritms found in implementations of Python, Java, and other libraries. The LLRB +// implementation of 2-3 trees is a recent improvement on the traditional implementation, +// observed and documented by Robert Sedgewick. +// +package llrb + +// Tree is a Left-Leaning Red-Black (LLRB) implementation of 2-3 trees +type LLRB struct { + count int + root *Node +} + +type Node struct { + Item + Left, Right *Node // Pointers to left and right child nodes + Black bool // If set, the color of the link (incoming from the parent) is black + // In the LLRB, new nodes are always red, hence the zero-value for node +} + +type Item interface { + Less(than Item) bool +} + +// +func less(x, y Item) bool { + if x == pinf { + return false + } + if x == ninf { + return true + } + return x.Less(y) +} + +// Inf returns an Item that is "bigger than" any other item, if sign is positive. +// Otherwise it returns an Item that is "smaller than" any other item. +func Inf(sign int) Item { + if sign == 0 { + panic("sign") + } + if sign > 0 { + return pinf + } + return ninf +} + +var ( + ninf = nInf{} + pinf = pInf{} +) + +type nInf struct{} + +func (nInf) Less(Item) bool { + return true +} + +type pInf struct{} + +func (pInf) Less(Item) bool { + return false +} + +// New() allocates a new tree +func New() *LLRB { + return &LLRB{} +} + +// SetRoot sets the root node of the tree. +// It is intended to be used by functions that deserialize the tree. +func (t *LLRB) SetRoot(r *Node) { + t.root = r +} + +// Root returns the root node of the tree. +// It is intended to be used by functions that serialize the tree. +func (t *LLRB) Root() *Node { + return t.root +} + +// Len returns the number of nodes in the tree. +func (t *LLRB) Len() int { return t.count } + +// Has returns true if the tree contains an element whose order is the same as that of key. +func (t *LLRB) Has(key Item) bool { + return t.Get(key) != nil +} + +// Get retrieves an element from the tree whose order is the same as that of key. +func (t *LLRB) Get(key Item) Item { + h := t.root + for h != nil { + switch { + case less(key, h.Item): + h = h.Left + case less(h.Item, key): + h = h.Right + default: + return h.Item + } + } + return nil +} + +// Min returns the minimum element in the tree. +func (t *LLRB) Min() Item { + h := t.root + if h == nil { + return nil + } + for h.Left != nil { + h = h.Left + } + return h.Item +} + +// Max returns the maximum element in the tree. +func (t *LLRB) Max() Item { + h := t.root + if h == nil { + return nil + } + for h.Right != nil { + h = h.Right + } + return h.Item +} + +func (t *LLRB) ReplaceOrInsertBulk(items ...Item) { + for _, i := range items { + t.ReplaceOrInsert(i) + } +} + +func (t *LLRB) InsertNoReplaceBulk(items ...Item) { + for _, i := range items { + t.InsertNoReplace(i) + } +} + +// ReplaceOrInsert inserts item into the tree. If an existing +// element has the same order, it is removed from the tree and returned. +func (t *LLRB) ReplaceOrInsert(item Item) Item { + if item == nil { + panic("inserting nil item") + } + var replaced Item + t.root, replaced = t.replaceOrInsert(t.root, item) + t.root.Black = true + if replaced == nil { + t.count++ + } + return replaced +} + +func (t *LLRB) replaceOrInsert(h *Node, item Item) (*Node, Item) { + if h == nil { + return newNode(item), nil + } + + h = walkDownRot23(h) + + var replaced Item + if less(item, h.Item) { // BUG + h.Left, replaced = t.replaceOrInsert(h.Left, item) + } else if less(h.Item, item) { + h.Right, replaced = t.replaceOrInsert(h.Right, item) + } else { + replaced, h.Item = h.Item, item + } + + h = walkUpRot23(h) + + return h, replaced +} + +// InsertNoReplace inserts item into the tree. If an existing +// element has the same order, both elements remain in the tree. +func (t *LLRB) InsertNoReplace(item Item) { + if item == nil { + panic("inserting nil item") + } + t.root = t.insertNoReplace(t.root, item) + t.root.Black = true + t.count++ +} + +func (t *LLRB) insertNoReplace(h *Node, item Item) *Node { + if h == nil { + return newNode(item) + } + + h = walkDownRot23(h) + + if less(item, h.Item) { + h.Left = t.insertNoReplace(h.Left, item) + } else { + h.Right = t.insertNoReplace(h.Right, item) + } + + return walkUpRot23(h) +} + +// Rotation driver routines for 2-3 algorithm + +func walkDownRot23(h *Node) *Node { return h } + +func walkUpRot23(h *Node) *Node { + if isRed(h.Right) && !isRed(h.Left) { + h = rotateLeft(h) + } + + if isRed(h.Left) && isRed(h.Left.Left) { + h = rotateRight(h) + } + + if isRed(h.Left) && isRed(h.Right) { + flip(h) + } + + return h +} + +// Rotation driver routines for 2-3-4 algorithm + +func walkDownRot234(h *Node) *Node { + if isRed(h.Left) && isRed(h.Right) { + flip(h) + } + + return h +} + +func walkUpRot234(h *Node) *Node { + if isRed(h.Right) && !isRed(h.Left) { + h = rotateLeft(h) + } + + if isRed(h.Left) && isRed(h.Left.Left) { + h = rotateRight(h) + } + + return h +} + +// DeleteMin deletes the minimum element in the tree and returns the +// deleted item or nil otherwise. +func (t *LLRB) DeleteMin() Item { + var deleted Item + t.root, deleted = deleteMin(t.root) + if t.root != nil { + t.root.Black = true + } + if deleted != nil { + t.count-- + } + return deleted +} + +// deleteMin code for LLRB 2-3 trees +func deleteMin(h *Node) (*Node, Item) { + if h == nil { + return nil, nil + } + if h.Left == nil { + return nil, h.Item + } + + if !isRed(h.Left) && !isRed(h.Left.Left) { + h = moveRedLeft(h) + } + + var deleted Item + h.Left, deleted = deleteMin(h.Left) + + return fixUp(h), deleted +} + +// DeleteMax deletes the maximum element in the tree and returns +// the deleted item or nil otherwise +func (t *LLRB) DeleteMax() Item { + var deleted Item + t.root, deleted = deleteMax(t.root) + if t.root != nil { + t.root.Black = true + } + if deleted != nil { + t.count-- + } + return deleted +} + +func deleteMax(h *Node) (*Node, Item) { + if h == nil { + return nil, nil + } + if isRed(h.Left) { + h = rotateRight(h) + } + if h.Right == nil { + return nil, h.Item + } + if !isRed(h.Right) && !isRed(h.Right.Left) { + h = moveRedRight(h) + } + var deleted Item + h.Right, deleted = deleteMax(h.Right) + + return fixUp(h), deleted +} + +// Delete deletes an item from the tree whose key equals key. +// The deleted item is return, otherwise nil is returned. +func (t *LLRB) Delete(key Item) Item { + var deleted Item + t.root, deleted = t.delete(t.root, key) + if t.root != nil { + t.root.Black = true + } + if deleted != nil { + t.count-- + } + return deleted +} + +func (t *LLRB) delete(h *Node, item Item) (*Node, Item) { + var deleted Item + if h == nil { + return nil, nil + } + if less(item, h.Item) { + if h.Left == nil { // item not present. Nothing to delete + return h, nil + } + if !isRed(h.Left) && !isRed(h.Left.Left) { + h = moveRedLeft(h) + } + h.Left, deleted = t.delete(h.Left, item) + } else { + if isRed(h.Left) { + h = rotateRight(h) + } + // If @item equals @h.Item and no right children at @h + if !less(h.Item, item) && h.Right == nil { + return nil, h.Item + } + // PETAR: Added 'h.Right != nil' below + if h.Right != nil && !isRed(h.Right) && !isRed(h.Right.Left) { + h = moveRedRight(h) + } + // If @item equals @h.Item, and (from above) 'h.Right != nil' + if !less(h.Item, item) { + var subDeleted Item + h.Right, subDeleted = deleteMin(h.Right) + if subDeleted == nil { + panic("logic") + } + deleted, h.Item = h.Item, subDeleted + } else { // Else, @item is bigger than @h.Item + h.Right, deleted = t.delete(h.Right, item) + } + } + + return fixUp(h), deleted +} + +// Internal node manipulation routines + +func newNode(item Item) *Node { return &Node{Item: item} } + +func isRed(h *Node) bool { + if h == nil { + return false + } + return !h.Black +} + +func rotateLeft(h *Node) *Node { + x := h.Right + if x.Black { + panic("rotating a black link") + } + h.Right = x.Left + x.Left = h + x.Black = h.Black + h.Black = false + return x +} + +func rotateRight(h *Node) *Node { + x := h.Left + if x.Black { + panic("rotating a black link") + } + h.Left = x.Right + x.Right = h + x.Black = h.Black + h.Black = false + return x +} + +// REQUIRE: Left and Right children must be present +func flip(h *Node) { + h.Black = !h.Black + h.Left.Black = !h.Left.Black + h.Right.Black = !h.Right.Black +} + +// REQUIRE: Left and Right children must be present +func moveRedLeft(h *Node) *Node { + flip(h) + if isRed(h.Right.Left) { + h.Right = rotateRight(h.Right) + h = rotateLeft(h) + flip(h) + } + return h +} + +// REQUIRE: Left and Right children must be present +func moveRedRight(h *Node) *Node { + flip(h) + if isRed(h.Left.Left) { + h = rotateRight(h) + flip(h) + } + return h +} + +func fixUp(h *Node) *Node { + if isRed(h.Right) { + h = rotateLeft(h) + } + + if isRed(h.Left) && isRed(h.Left.Left) { + h = rotateRight(h) + } + + if isRed(h.Left) && isRed(h.Right) { + flip(h) + } + + return h +} diff --git a/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb_test.go b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb_test.go new file mode 100644 index 00000000000..b7bc9780070 --- /dev/null +++ b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/llrb_test.go @@ -0,0 +1,239 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +import ( + "math" + "math/rand" + "testing" +) + +func TestCases(t *testing.T) { + tree := New() + tree.ReplaceOrInsert(Int(1)) + tree.ReplaceOrInsert(Int(1)) + if tree.Len() != 1 { + t.Errorf("expecting len 1") + } + if !tree.Has(Int(1)) { + t.Errorf("expecting to find key=1") + } + + tree.Delete(Int(1)) + if tree.Len() != 0 { + t.Errorf("expecting len 0") + } + if tree.Has(Int(1)) { + t.Errorf("not expecting to find key=1") + } + + tree.Delete(Int(1)) + if tree.Len() != 0 { + t.Errorf("expecting len 0") + } + if tree.Has(Int(1)) { + t.Errorf("not expecting to find key=1") + } +} + +func TestReverseInsertOrder(t *testing.T) { + tree := New() + n := 100 + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(n - i)) + } + i := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + i++ + if item.(Int) != Int(i) { + t.Errorf("bad order: got %d, expect %d", item.(Int), i) + } + return true + }) +} + +func TestRange(t *testing.T) { + tree := New() + order := []String{ + "ab", "aba", "abc", "a", "aa", "aaa", "b", "a-", "a!", + } + for _, i := range order { + tree.ReplaceOrInsert(i) + } + k := 0 + tree.AscendRange(String("ab"), String("ac"), func(item Item) bool { + if k > 3 { + t.Fatalf("returned more items than expected") + } + i1 := order[k] + i2 := item.(String) + if i1 != i2 { + t.Errorf("expecting %s, got %s", i1, i2) + } + k++ + return true + }) +} + +func TestRandomInsertOrder(t *testing.T) { + tree := New() + n := 1000 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + j := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + if item.(Int) != Int(j) { + t.Fatalf("bad order") + } + j++ + return true + }) +} + +func TestRandomReplace(t *testing.T) { + tree := New() + n := 100 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + perm = rand.Perm(n) + for i := 0; i < n; i++ { + if replaced := tree.ReplaceOrInsert(Int(perm[i])); replaced == nil || replaced.(Int) != Int(perm[i]) { + t.Errorf("error replacing") + } + } +} + +func TestRandomInsertSequentialDelete(t *testing.T) { + tree := New() + n := 1000 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + for i := 0; i < n; i++ { + tree.Delete(Int(i)) + } +} + +func TestRandomInsertDeleteNonExistent(t *testing.T) { + tree := New() + n := 100 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + if tree.Delete(Int(200)) != nil { + t.Errorf("deleted non-existent item") + } + if tree.Delete(Int(-2)) != nil { + t.Errorf("deleted non-existent item") + } + for i := 0; i < n; i++ { + if u := tree.Delete(Int(i)); u == nil || u.(Int) != Int(i) { + t.Errorf("delete failed") + } + } + if tree.Delete(Int(200)) != nil { + t.Errorf("deleted non-existent item") + } + if tree.Delete(Int(-2)) != nil { + t.Errorf("deleted non-existent item") + } +} + +func TestRandomInsertPartialDeleteOrder(t *testing.T) { + tree := New() + n := 100 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + for i := 1; i < n-1; i++ { + tree.Delete(Int(i)) + } + j := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + switch j { + case 0: + if item.(Int) != Int(0) { + t.Errorf("expecting 0") + } + case 1: + if item.(Int) != Int(n-1) { + t.Errorf("expecting %d", n-1) + } + } + j++ + return true + }) +} + +func TestRandomInsertStats(t *testing.T) { + tree := New() + n := 100000 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + avg, _ := tree.HeightStats() + expAvg := math.Log2(float64(n)) - 1.5 + if math.Abs(avg-expAvg) >= 2.0 { + t.Errorf("too much deviation from expected average height") + } +} + +func BenchmarkInsert(b *testing.B) { + tree := New() + for i := 0; i < b.N; i++ { + tree.ReplaceOrInsert(Int(b.N - i)) + } +} + +func BenchmarkDelete(b *testing.B) { + b.StopTimer() + tree := New() + for i := 0; i < b.N; i++ { + tree.ReplaceOrInsert(Int(b.N - i)) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + tree.Delete(Int(i)) + } +} + +func BenchmarkDeleteMin(b *testing.B) { + b.StopTimer() + tree := New() + for i := 0; i < b.N; i++ { + tree.ReplaceOrInsert(Int(b.N - i)) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + tree.DeleteMin() + } +} + +func TestInsertNoReplace(t *testing.T) { + tree := New() + n := 1000 + for q := 0; q < 2; q++ { + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.InsertNoReplace(Int(perm[i])) + } + } + j := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + if item.(Int) != Int(j/2) { + t.Fatalf("bad order") + } + j++ + return true + }) +} diff --git a/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/util.go b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/util.go new file mode 100644 index 00000000000..63dbdb2df0a --- /dev/null +++ b/Godeps/_workspace/src/github.com/petar/GoLLRB/llrb/util.go @@ -0,0 +1,17 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +type Int int + +func (x Int) Less(than Item) bool { + return x < than.(Int) +} + +type String string + +func (x String) Less(than Item) bool { + return x < than.(String) +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/LICENSE b/Godeps/_workspace/src/github.com/peterbourgon/diskv/LICENSE new file mode 100644 index 00000000000..41ce7f16e1d --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2011-2012 Peter Bourgon + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/README.md b/Godeps/_workspace/src/github.com/peterbourgon/diskv/README.md new file mode 100644 index 00000000000..ed59b7bfb68 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/README.md @@ -0,0 +1,141 @@ +# What is diskv? + +Diskv (disk-vee) is a simple, persistent key-value store written in the Go +language. It starts with an incredibly simple API for storing arbitrary data on +a filesystem by key, and builds several layers of performance-enhancing +abstraction on top. The end result is a conceptually simple, but highly +performant, disk-backed storage system. + +[![Build Status][1]][2] + +[1]: https://drone.io/github.com/peterbourgon/diskv/status.png +[2]: https://drone.io/github.com/peterbourgon/diskv/latest + + +# Installing + +Install [Go 1][3], either [from source][4] or [with a prepackaged binary][5]. +Then, + +```bash +$ go get github.com/peterbourgon/diskv +``` + +[3]: http://golang.org +[4]: http://golang.org/doc/install/source +[5]: http://golang.org/doc/install + + +# Usage + +```go +package main + +import ( + "fmt" + "github.com/peterbourgon/diskv" +) + +func main() { + // Simplest transform function: put all the data files into the base dir. + flatTransform := func(s string) []string { return []string{} } + + // Initialize a new diskv store, rooted at "my-data-dir", with a 1MB cache. + d := diskv.New(diskv.Options{ + BasePath: "my-data-dir", + Transform: flatTransform, + CacheSizeMax: 1024 * 1024, + }) + + // Write three bytes to the key "alpha". + key := "alpha" + d.Write(key, []byte{'1', '2', '3'}) + + // Read the value back out of the store. + value, _ := d.Read(key) + fmt.Printf("%v\n", value) + + // Erase the key+value from the store (and the disk). + d.Erase(key) +} +``` + +More complex examples can be found in the "examples" subdirectory. + + +# Theory + +## Basic idea + +At its core, diskv is a map of a key (`string`) to arbitrary data (`[]byte`). +The data is written to a single file on disk, with the same name as the key. +The key determines where that file will be stored, via a user-provided +`TransformFunc`, which takes a key and returns a slice (`[]string`) +corresponding to a path list where the key file will be stored. The simplest +TransformFunc, + +```go +func SimpleTransform (key string) []string { + return []string{} +} +``` + +will place all keys in the same, base directory. The design is inspired by +[Redis diskstore][6]; a TransformFunc which emulates the default diskstore +behavior is available in the content-addressable-storage example. + +[6]: http://groups.google.com/group/redis-db/browse_thread/thread/d444bc786689bde9?pli=1 + +**Note** that your TransformFunc should ensure that one valid key doesn't +transform to a subset of another valid key. That is, it shouldn't be possible +to construct valid keys that resolve to directory names. As a concrete example, +if your TransformFunc splits on every 3 characters, then + +```go +d.Write("abcabc", val) // OK: written to /abc/abc/abcabc +d.Write("abc", val) // Error: attempted write to /abc/abc, but it's a directory +``` + +This will be addressed in an upcoming version of diskv. + +Probably the most important design principle behind diskv is that your data is +always flatly available on the disk. diskv will never do anything that would +prevent you from accessing, copying, backing up, or otherwise interacting with +your data via common UNIX commandline tools. + +## Adding a cache + +An in-memory caching layer is provided by combining the BasicStore +functionality with a simple map structure, and keeping it up-to-date as +appropriate. Since the map structure in Go is not threadsafe, it's combined +with a RWMutex to provide safe concurrent access. + +## Adding order + +diskv is a key-value store and therefore inherently unordered. An ordering +system can be injected into the store by passing something which satisfies the +diskv.Index interface. (A default implementation, using Petar Maymounkov's +[LLRB tree][7], is provided.) Basically, diskv keeps an ordered (by a +user-provided Less function) index of the keys, which can be queried. + +[7]: https://github.com/petar/GoLLRB + +## Adding compression + +Something which implements the diskv.Compression interface may be passed +during store creation, so that all Writes and Reads are filtered through +a compression/decompression pipeline. Several default implementations, +using stdlib compression algorithms, are provided. Note that data is cached +compressed; the cost of decompression is borne with each Read. + +## Streaming + +diskv also now provides ReadStream and WriteStream methods, to allow very large +data to be handled efficiently. + + +# Future plans + + * Needs plenty of robust testing: huge datasets, etc... + * More thorough benchmarking + * Your suggestions for use-cases I haven't thought of diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/basic_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/basic_test.go new file mode 100644 index 00000000000..ef0d32fd261 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/basic_test.go @@ -0,0 +1,253 @@ +package diskv + +import ( + "bytes" + "testing" + "time" +) + +func cmpBytes(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func (d *Diskv) isCached(key string) bool { + d.RLock() + defer d.RUnlock() + _, ok := d.cache[key] + return ok +} + +func TestWriteReadErase(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 1024, + }) + defer d.EraseAll() + k, v := "a", []byte{'b'} + if err := d.Write(k, v); err != nil { + t.Fatalf("write: %s", err) + } + if readVal, err := d.Read(k); err != nil { + t.Fatalf("read: %s", err) + } else if bytes.Compare(v, readVal) != 0 { + t.Fatalf("read: expected %s, got %s", v, readVal) + } + if err := d.Erase(k); err != nil { + t.Fatalf("erase: %s", err) + } +} + +func TestWRECache(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 1024, + }) + defer d.EraseAll() + k, v := "xxx", []byte{' ', ' ', ' '} + if d.isCached(k) { + t.Fatalf("key cached before Write and Read") + } + if err := d.Write(k, v); err != nil { + t.Fatalf("write: %s", err) + } + if d.isCached(k) { + t.Fatalf("key cached before Read") + } + if readVal, err := d.Read(k); err != nil { + t.Fatalf("read: %s", err) + } else if bytes.Compare(v, readVal) != 0 { + t.Fatalf("read: expected %s, got %s", v, readVal) + } + for i := 0; i < 10 && !d.isCached(k); i++ { + time.Sleep(10 * time.Millisecond) + } + if !d.isCached(k) { + t.Fatalf("key not cached after Read") + } + if err := d.Erase(k); err != nil { + t.Fatalf("erase: %s", err) + } + if d.isCached(k) { + t.Fatalf("key cached after Erase") + } +} + +func TestStrings(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 1024, + }) + defer d.EraseAll() + + keys := map[string]bool{"a": false, "b": false, "c": false, "d": false} + v := []byte{'1'} + for k := range keys { + if err := d.Write(k, v); err != nil { + t.Fatalf("write: %s: %s", k, err) + } + } + + for k := range d.Keys(nil) { + if _, present := keys[k]; present { + t.Logf("got: %s", k) + keys[k] = true + } else { + t.Fatalf("strings() returns unknown key: %s", k) + } + } + + for k, found := range keys { + if !found { + t.Errorf("never got %s", k) + } + } +} + +func TestZeroByteCache(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 0, + }) + defer d.EraseAll() + + k, v := "a", []byte{'1', '2', '3'} + if err := d.Write(k, v); err != nil { + t.Fatalf("Write: %s", err) + } + + if d.isCached(k) { + t.Fatalf("key cached, expected not-cached") + } + + if _, err := d.Read(k); err != nil { + t.Fatalf("Read: %s", err) + } + + if d.isCached(k) { + t.Fatalf("key cached, expected not-cached") + } +} + +func TestOneByteCache(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 1, + }) + defer d.EraseAll() + + k1, k2, v1, v2 := "a", "b", []byte{'1'}, []byte{'1', '2'} + if err := d.Write(k1, v1); err != nil { + t.Fatal(err) + } + + if v, err := d.Read(k1); err != nil { + t.Fatal(err) + } else if !cmpBytes(v, v1) { + t.Fatalf("Read: expected %s, got %s", string(v1), string(v)) + } + + for i := 0; i < 10 && !d.isCached(k1); i++ { + time.Sleep(10 * time.Millisecond) + } + if !d.isCached(k1) { + t.Fatalf("expected 1-byte value to be cached, but it wasn't") + } + + if err := d.Write(k2, v2); err != nil { + t.Fatal(err) + } + if _, err := d.Read(k2); err != nil { + t.Fatalf("--> %s", err) + } + + for i := 0; i < 10 && (!d.isCached(k1) || d.isCached(k2)); i++ { + time.Sleep(10 * time.Millisecond) // just wait for lazy-cache + } + if !d.isCached(k1) { + t.Fatalf("1-byte value was uncached for no reason") + } + + if d.isCached(k2) { + t.Fatalf("2-byte value was cached, but cache max size is 1") + } +} + +func TestStaleCache(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 1, + }) + defer d.EraseAll() + + k, first, second := "a", "first", "second" + if err := d.Write(k, []byte(first)); err != nil { + t.Fatal(err) + } + + v, err := d.Read(k) + if err != nil { + t.Fatal(err) + } + if string(v) != first { + t.Errorf("expected '%s', got '%s'", first, v) + } + + if err := d.Write(k, []byte(second)); err != nil { + t.Fatal(err) + } + + v, err = d.Read(k) + if err != nil { + t.Fatal(err) + } + + if string(v) != second { + t.Errorf("expected '%s', got '%s'", second, v) + } +} + +func TestHas(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 1024, + }) + defer d.EraseAll() + + for k, v := range map[string]string{ + "a": "1", + "foo": "2", + "012345": "3", + } { + d.Write(k, []byte(v)) + } + + d.Read("foo") // cache one of them + if !d.isCached("foo") { + t.Errorf("'foo' didn't get cached") + } + + for _, tuple := range []struct { + key string + expected bool + }{ + {"a", true}, + {"b", false}, + {"foo", true}, + {"bar", false}, + {"01234", false}, + {"012345", true}, + {"0123456", false}, + } { + if expected, got := tuple.expected, d.Has(tuple.key); expected != got { + t.Errorf("Has(%s): expected %v, got %v", tuple.key, expected, got) + } + } +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/compression.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/compression.go new file mode 100644 index 00000000000..5192b027330 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/compression.go @@ -0,0 +1,64 @@ +package diskv + +import ( + "compress/flate" + "compress/gzip" + "compress/zlib" + "io" +) + +// Compression is an interface that Diskv uses to implement compression of +// data. Writer takes a destination io.Writer and returns a WriteCloser that +// compresses all data written through it. Reader takes a source io.Reader and +// returns a ReadCloser that decompresses all data read through it. You may +// define these methods on your own type, or use one of the NewCompression +// helpers. +type Compression interface { + Writer(dst io.Writer) (io.WriteCloser, error) + Reader(src io.Reader) (io.ReadCloser, error) +} + +// NewGzipCompression returns a Gzip-based Compression. +func NewGzipCompression() Compression { + return NewGzipCompressionLevel(flate.DefaultCompression) +} + +// NewGzipCompressionLevel returns a Gzip-based Compression with the given level. +func NewGzipCompressionLevel(level int) Compression { + return &genericCompression{ + wf: func(w io.Writer) (io.WriteCloser, error) { return gzip.NewWriterLevel(w, level) }, + rf: func(r io.Reader) (io.ReadCloser, error) { return gzip.NewReader(r) }, + } +} + +// NewZlibCompression returns a Zlib-based Compression. +func NewZlibCompression() Compression { + return NewZlibCompressionLevel(flate.DefaultCompression) +} + +// NewZlibCompressionLevel returns a Zlib-based Compression with the given level. +func NewZlibCompressionLevel(level int) Compression { + return NewZlibCompressionLevelDict(level, nil) +} + +// NewZlibCompressionLevelDict returns a Zlib-based Compression with the given +// level, based on the given dictionary. +func NewZlibCompressionLevelDict(level int, dict []byte) Compression { + return &genericCompression{ + func(w io.Writer) (io.WriteCloser, error) { return zlib.NewWriterLevelDict(w, level, dict) }, + func(r io.Reader) (io.ReadCloser, error) { return zlib.NewReaderDict(r, dict) }, + } +} + +type genericCompression struct { + wf func(w io.Writer) (io.WriteCloser, error) + rf func(r io.Reader) (io.ReadCloser, error) +} + +func (g *genericCompression) Writer(dst io.Writer) (io.WriteCloser, error) { + return g.wf(dst) +} + +func (g *genericCompression) Reader(src io.Reader) (io.ReadCloser, error) { + return g.rf(src) +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/compression_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/compression_test.go new file mode 100644 index 00000000000..2d614203725 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/compression_test.go @@ -0,0 +1,72 @@ +package diskv + +import ( + "compress/flate" + "fmt" + "math/rand" + "os" + "testing" + "time" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func testCompressionWith(t *testing.T, c Compression, name string) { + d := New(Options{ + BasePath: "compression-test", + CacheSizeMax: 0, + Compression: c, + }) + defer d.EraseAll() + + sz := 4096 + val := make([]byte, sz) + for i := 0; i < sz; i++ { + val[i] = byte('a' + rand.Intn(26)) // {a-z}; should compress some + } + + key := "a" + if err := d.Write(key, val); err != nil { + t.Fatalf("write failed: %s", err) + } + + targetFile := fmt.Sprintf("%s%c%s", d.BasePath, os.PathSeparator, key) + fi, err := os.Stat(targetFile) + if err != nil { + t.Fatalf("%s: %s", targetFile, err) + } + + if fi.Size() >= int64(sz) { + t.Fatalf("%s: size=%d, expected smaller", targetFile, fi.Size()) + } + t.Logf("%s compressed %d to %d", name, sz, fi.Size()) + + readVal, err := d.Read(key) + if len(readVal) != sz { + t.Fatalf("read: expected size=%d, got size=%d", sz, len(readVal)) + } + + for i := 0; i < sz; i++ { + if readVal[i] != val[i] { + t.Fatalf("i=%d: expected %v, got %v", i, val[i], readVal[i]) + } + } +} + +func TestGzipDefault(t *testing.T) { + testCompressionWith(t, NewGzipCompression(), "gzip") +} + +func TestGzipBestCompression(t *testing.T) { + testCompressionWith(t, NewGzipCompressionLevel(flate.BestCompression), "gzip-max") +} + +func TestGzipBestSpeed(t *testing.T) { + testCompressionWith(t, NewGzipCompressionLevel(flate.BestSpeed), "gzip-min") +} + +func TestZlib(t *testing.T) { + testCompressionWith(t, NewZlibCompression(), "zlib") +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/diskv.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/diskv.go new file mode 100644 index 00000000000..512b5353fd2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/diskv.go @@ -0,0 +1,576 @@ +// Diskv (disk-vee) is a simple, persistent, key-value store. +// It stores all data flatly on the filesystem. + +package diskv + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "syscall" +) + +const ( + defaultBasePath = "diskv" + defaultFilePerm os.FileMode = 0666 + defaultPathPerm os.FileMode = 0777 +) + +var ( + defaultTransform = func(s string) []string { return []string{} } + errCanceled = errors.New("canceled") + errEmptyKey = errors.New("empty key") + errBadKey = errors.New("bad key") + errImportDirectory = errors.New("can't import a directory") +) + +// TransformFunction transforms a key into a slice of strings, with each +// element in the slice representing a directory in the file path where the +// key's entry will eventually be stored. +// +// For example, if TransformFunc transforms "abcdef" to ["ab", "cde", "f"], +// the final location of the data file will be /ab/cde/f/abcdef +type TransformFunction func(s string) []string + +// Options define a set of properties that dictate Diskv behavior. +// All values are optional. +type Options struct { + BasePath string + Transform TransformFunction + CacheSizeMax uint64 // bytes + PathPerm os.FileMode + FilePerm os.FileMode + + Index Index + IndexLess LessFunction + + Compression Compression +} + +// Diskv implements the Diskv interface. You shouldn't construct Diskv +// structures directly; instead, use the New constructor. +type Diskv struct { + sync.RWMutex + Options + cache map[string][]byte + cacheSize uint64 +} + +// New returns an initialized Diskv structure, ready to use. +// If the path identified by baseDir already contains data, +// it will be accessible, but not yet cached. +func New(o Options) *Diskv { + if o.BasePath == "" { + o.BasePath = defaultBasePath + } + if o.Transform == nil { + o.Transform = defaultTransform + } + if o.PathPerm == 0 { + o.PathPerm = defaultPathPerm + } + if o.FilePerm == 0 { + o.FilePerm = defaultFilePerm + } + + d := &Diskv{ + Options: o, + cache: map[string][]byte{}, + cacheSize: 0, + } + + if d.Index != nil && d.IndexLess != nil { + d.Index.Initialize(d.IndexLess, d.Keys(nil)) + } + + return d +} + +// Write synchronously writes the key-value pair to disk, making it immediately +// available for reads. Write relies on the filesystem to perform an eventual +// sync to physical media. If you need stronger guarantees, see WriteStream. +func (d *Diskv) Write(key string, val []byte) error { + return d.WriteStream(key, bytes.NewBuffer(val), false) +} + +// WriteStream writes the data represented by the io.Reader to the disk, under +// the provided key. If sync is true, WriteStream performs an explicit sync on +// the file as soon as it's written. +// +// bytes.Buffer provides io.Reader semantics for basic data types. +func (d *Diskv) WriteStream(key string, r io.Reader, sync bool) error { + if len(key) <= 0 { + return errEmptyKey + } + + d.Lock() + defer d.Unlock() + + return d.writeStreamWithLock(key, r, sync) +} + +// writeStream does no input validation checking. +// TODO: use atomic FS ops. +func (d *Diskv) writeStreamWithLock(key string, r io.Reader, sync bool) error { + if err := d.ensurePathWithLock(key); err != nil { + return fmt.Errorf("ensure path: %s", err) + } + + mode := os.O_WRONLY | os.O_CREATE | os.O_TRUNC // overwrite if exists + f, err := os.OpenFile(d.completeFilename(key), mode, d.FilePerm) + if err != nil { + return fmt.Errorf("open file: %s", err) + } + + wc := io.WriteCloser(&nopWriteCloser{f}) + if d.Compression != nil { + wc, err = d.Compression.Writer(f) + if err != nil { + f.Close() // error deliberately ignored + return fmt.Errorf("compression writer: %s", err) + } + } + + if _, err := io.Copy(wc, r); err != nil { + f.Close() // error deliberately ignored + return fmt.Errorf("i/o copy: %s", err) + } + + if err := wc.Close(); err != nil { + return fmt.Errorf("compression close: %s", err) + } + + if sync { + if err := f.Sync(); err != nil { + f.Close() // error deliberately ignored + return fmt.Errorf("file sync: %s", err) + } + } + + if err := f.Close(); err != nil { + return fmt.Errorf("file close: %s", err) + } + + if d.Index != nil { + d.Index.Insert(key) + } + + d.bustCacheWithLock(key) // cache only on read + + return nil +} + +// Import imports the source file into diskv under the destination key. If the +// destination key already exists, it's overwritten. If move is true, the +// source file is removed after a successful import. +func (d *Diskv) Import(srcFilename, dstKey string, move bool) (err error) { + if dstKey == "" { + return errEmptyKey + } + + if fi, err := os.Stat(srcFilename); err != nil { + return err + } else if fi.IsDir() { + return errImportDirectory + } + + d.Lock() + defer d.Unlock() + + if err := d.ensurePathWithLock(dstKey); err != nil { + return fmt.Errorf("ensure path: %s", err) + } + + if move { + if err := syscall.Rename(srcFilename, d.completeFilename(dstKey)); err == nil { + d.bustCacheWithLock(dstKey) + return nil + } else if err != syscall.EXDEV { + // If it failed due to being on a different device, fall back to copying + return err + } + } + + f, err := os.Open(srcFilename) + if err != nil { + return err + } + defer f.Close() + err = d.writeStreamWithLock(dstKey, f, false) + if err == nil && move { + err = os.Remove(srcFilename) + } + return err +} + +// Read reads the key and returns the value. +// If the key is available in the cache, Read won't touch the disk. +// If the key is not in the cache, Read will have the side-effect of +// lazily caching the value. +func (d *Diskv) Read(key string) ([]byte, error) { + rc, err := d.ReadStream(key, false) + if err != nil { + return []byte{}, err + } + defer rc.Close() + return ioutil.ReadAll(rc) +} + +// ReadStream reads the key and returns the value (data) as an io.ReadCloser. +// If the value is cached from a previous read, and direct is false, +// ReadStream will use the cached value. Otherwise, it will return a handle to +// the file on disk, and cache the data on read. +// +// If direct is true, ReadStream will lazily delete any cached value for the +// key, and return a direct handle to the file on disk. +// +// If compression is enabled, ReadStream taps into the io.Reader stream prior +// to decompression, and caches the compressed data. +func (d *Diskv) ReadStream(key string, direct bool) (io.ReadCloser, error) { + d.RLock() + defer d.RUnlock() + + if val, ok := d.cache[key]; ok { + if !direct { + buf := bytes.NewBuffer(val) + if d.Compression != nil { + return d.Compression.Reader(buf) + } + return ioutil.NopCloser(buf), nil + } + + go func() { + d.Lock() + defer d.Unlock() + d.uncacheWithLock(key, uint64(len(val))) + }() + } + + return d.readWithRLock(key) +} + +// read ignores the cache, and returns an io.ReadCloser representing the +// decompressed data for the given key, streamed from the disk. Clients should +// acquire a read lock on the Diskv and check the cache themselves before +// calling read. +func (d *Diskv) readWithRLock(key string) (io.ReadCloser, error) { + filename := d.completeFilename(key) + + fi, err := os.Stat(filename) + if err != nil { + return nil, err + } + if fi.IsDir() { + return nil, os.ErrNotExist + } + + f, err := os.Open(filename) + if err != nil { + return nil, err + } + + var r io.Reader + if d.CacheSizeMax > 0 { + r = newSiphon(f, d, key) + } else { + r = &closingReader{f} + } + + var rc = io.ReadCloser(ioutil.NopCloser(r)) + if d.Compression != nil { + rc, err = d.Compression.Reader(r) + if err != nil { + return nil, err + } + } + + return rc, nil +} + +// closingReader provides a Reader that automatically closes the +// embedded ReadCloser when it reaches EOF +type closingReader struct { + rc io.ReadCloser +} + +func (cr closingReader) Read(p []byte) (int, error) { + n, err := cr.rc.Read(p) + if err == io.EOF { + if closeErr := cr.rc.Close(); closeErr != nil { + return n, closeErr // close must succeed for Read to succeed + } + } + return n, err +} + +// siphon is like a TeeReader: it copies all data read through it to an +// internal buffer, and moves that buffer to the cache at EOF. +type siphon struct { + f *os.File + d *Diskv + key string + buf *bytes.Buffer +} + +// newSiphon constructs a siphoning reader that represents the passed file. +// When a successful series of reads ends in an EOF, the siphon will write +// the buffered data to Diskv's cache under the given key. +func newSiphon(f *os.File, d *Diskv, key string) io.Reader { + return &siphon{ + f: f, + d: d, + key: key, + buf: &bytes.Buffer{}, + } +} + +// Read implements the io.Reader interface for siphon. +func (s *siphon) Read(p []byte) (int, error) { + n, err := s.f.Read(p) + + if err == nil { + return s.buf.Write(p[0:n]) // Write must succeed for Read to succeed + } + + if err == io.EOF { + s.d.cacheWithoutLock(s.key, s.buf.Bytes()) // cache may fail + if closeErr := s.f.Close(); closeErr != nil { + return n, closeErr // close must succeed for Read to succeed + } + return n, err + } + + return n, err +} + +// Erase synchronously erases the given key from the disk and the cache. +func (d *Diskv) Erase(key string) error { + d.Lock() + defer d.Unlock() + + d.bustCacheWithLock(key) + + // erase from index + if d.Index != nil { + d.Index.Delete(key) + } + + // erase from disk + filename := d.completeFilename(key) + if s, err := os.Stat(filename); err == nil { + if !!s.IsDir() { + return errBadKey + } + if err = os.Remove(filename); err != nil { + return fmt.Errorf("remove: %s", err) + } + } else { + return fmt.Errorf("stat: %s", err) + } + + // clean up and return + d.pruneDirsWithLock(key) + return nil +} + +// EraseAll will delete all of the data from the store, both in the cache and on +// the disk. Note that EraseAll doesn't distinguish diskv-related data from non- +// diskv-related data. Care should be taken to always specify a diskv base +// directory that is exclusively for diskv data. +func (d *Diskv) EraseAll() error { + d.Lock() + defer d.Unlock() + d.cache = make(map[string][]byte) + d.cacheSize = 0 + return os.RemoveAll(d.BasePath) +} + +// Has returns true if the given key exists. +func (d *Diskv) Has(key string) bool { + d.Lock() + defer d.Unlock() + + if _, ok := d.cache[key]; ok { + return true + } + + filename := d.completeFilename(key) + s, err := os.Stat(filename) + if err != nil { + return false + } + if s.IsDir() { + return false + } + + return true +} + +// Keys returns a channel that will yield every key accessible by the store, +// in undefined order. If a cancel channel is provided, closing it will +// terminate and close the keys channel. +func (d *Diskv) Keys(cancel <-chan struct{}) <-chan string { + return d.KeysPrefix("", cancel) +} + +// KeysPrefix returns a channel that will yield every key accessible by the +// store with the given prefix, in undefined order. If a cancel channel is +// provided, closing it will terminate and close the keys channel. If the +// provided prefix is the empty string, all keys will be yielded. +func (d *Diskv) KeysPrefix(prefix string, cancel <-chan struct{}) <-chan string { + var prepath string + if prefix == "" { + prepath = d.BasePath + } else { + prepath = d.pathFor(prefix) + } + c := make(chan string) + go func() { + filepath.Walk(prepath, walker(c, prefix, cancel)) + close(c) + }() + return c +} + +// walker returns a function which satisfies the filepath.WalkFunc interface. +// It sends every non-directory file entry down the channel c. +func walker(c chan<- string, prefix string, cancel <-chan struct{}) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() || !strings.HasPrefix(info.Name(), prefix) { + return nil // "pass" + } + + select { + case c <- info.Name(): + case <-cancel: + return errCanceled + } + + return nil + } +} + +// pathFor returns the absolute path for location on the filesystem where the +// data for the given key will be stored. +func (d *Diskv) pathFor(key string) string { + return filepath.Join(d.BasePath, filepath.Join(d.Transform(key)...)) +} + +// ensurePathWithLock is a helper function that generates all necessary +// directories on the filesystem for the given key. +func (d *Diskv) ensurePathWithLock(key string) error { + return os.MkdirAll(d.pathFor(key), d.PathPerm) +} + +// completeFilename returns the absolute path to the file for the given key. +func (d *Diskv) completeFilename(key string) string { + return filepath.Join(d.pathFor(key), key) +} + +// cacheWithLock attempts to cache the given key-value pair in the store's +// cache. It can fail if the value is larger than the cache's maximum size. +func (d *Diskv) cacheWithLock(key string, val []byte) error { + valueSize := uint64(len(val)) + if err := d.ensureCacheSpaceWithLock(valueSize); err != nil { + return fmt.Errorf("%s; not caching", err) + } + + // be very strict about memory guarantees + if (d.cacheSize + valueSize) > d.CacheSizeMax { + panic(fmt.Sprintf("failed to make room for value (%d/%d)", valueSize, d.CacheSizeMax)) + } + + d.cache[key] = val + d.cacheSize += valueSize + return nil +} + +// cacheWithoutLock acquires the store's (write) mutex and calls cacheWithLock. +func (d *Diskv) cacheWithoutLock(key string, val []byte) error { + d.Lock() + defer d.Unlock() + return d.cacheWithLock(key, val) +} + +func (d *Diskv) bustCacheWithLock(key string) { + if val, ok := d.cache[key]; ok { + d.uncacheWithLock(key, uint64(len(val))) + } +} + +func (d *Diskv) uncacheWithLock(key string, sz uint64) { + d.cacheSize -= sz + delete(d.cache, key) +} + +// pruneDirsWithLock deletes empty directories in the path walk leading to the +// key k. Typically this function is called after an Erase is made. +func (d *Diskv) pruneDirsWithLock(key string) error { + pathlist := d.Transform(key) + for i := range pathlist { + dir := filepath.Join(d.BasePath, filepath.Join(pathlist[:len(pathlist)-i]...)) + + // thanks to Steven Blenkinsop for this snippet + switch fi, err := os.Stat(dir); true { + case err != nil: + return err + case !fi.IsDir(): + panic(fmt.Sprintf("corrupt dirstate at %s", dir)) + } + + nlinks, err := filepath.Glob(filepath.Join(dir, "*")) + if err != nil { + return err + } else if len(nlinks) > 0 { + return nil // has subdirs -- do not prune + } + if err = os.Remove(dir); err != nil { + return err + } + } + + return nil +} + +// ensureCacheSpaceWithLock deletes entries from the cache in arbitrary order +// until the cache has at least valueSize bytes available. +func (d *Diskv) ensureCacheSpaceWithLock(valueSize uint64) error { + if valueSize > d.CacheSizeMax { + return fmt.Errorf("value size (%d bytes) too large for cache (%d bytes)", valueSize, d.CacheSizeMax) + } + + safe := func() bool { return (d.cacheSize + valueSize) <= d.CacheSizeMax } + + for key, val := range d.cache { + if safe() { + break + } + + d.uncacheWithLock(key, uint64(len(val))) + } + + if !safe() { + panic(fmt.Sprintf("%d bytes still won't fit in the cache! (max %d bytes)", valueSize, d.CacheSizeMax)) + } + + return nil +} + +// nopWriteCloser wraps an io.Writer and provides a no-op Close method to +// satisfy the io.WriteCloser interface. +type nopWriteCloser struct { + io.Writer +} + +func (wc *nopWriteCloser) Write(p []byte) (int, error) { return wc.Writer.Write(p) } +func (wc *nopWriteCloser) Close() error { return nil } diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/examples/content-addressable-store/cas.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/examples/content-addressable-store/cas.go new file mode 100644 index 00000000000..a3abaaf7750 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/examples/content-addressable-store/cas.go @@ -0,0 +1,63 @@ +package main + +import ( + "crypto/md5" + "fmt" + "io" + + "github.com/peterbourgon/diskv" +) + +const transformBlockSize = 2 // grouping of chars per directory depth + +func blockTransform(s string) []string { + var ( + sliceSize = len(s) / transformBlockSize + pathSlice = make([]string, sliceSize) + ) + for i := 0; i < sliceSize; i++ { + from, to := i*transformBlockSize, (i*transformBlockSize)+transformBlockSize + pathSlice[i] = s[from:to] + } + return pathSlice +} + +func main() { + d := diskv.New(diskv.Options{ + BasePath: "data", + Transform: blockTransform, + CacheSizeMax: 1024 * 1024, // 1MB + }) + + for _, valueStr := range []string{ + "I am the very model of a modern Major-General", + "I've information vegetable, animal, and mineral", + "I know the kings of England, and I quote the fights historical", + "From Marathon to Waterloo, in order categorical", + "I'm very well acquainted, too, with matters mathematical", + "I understand equations, both the simple and quadratical", + "About binomial theorem I'm teeming with a lot o' news", + "With many cheerful facts about the square of the hypotenuse", + } { + d.Write(md5sum(valueStr), []byte(valueStr)) + } + + var keyCount int + for key := range d.Keys(nil) { + val, err := d.Read(key) + if err != nil { + panic(fmt.Sprintf("key %s had no value", key)) + } + fmt.Printf("%s: %s\n", key, val) + keyCount++ + } + fmt.Printf("%d total keys\n", keyCount) + + // d.EraseAll() // leave it commented out to see how data is kept on disk +} + +func md5sum(s string) string { + h := md5.New() + io.WriteString(h, s) + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/examples/super-simple-store/super-simple-store.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/examples/super-simple-store/super-simple-store.go new file mode 100644 index 00000000000..b5da11d6464 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/examples/super-simple-store/super-simple-store.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + + "github.com/peterbourgon/diskv" +) + +func main() { + d := diskv.New(diskv.Options{ + BasePath: "my-diskv-data-directory", + Transform: func(s string) []string { return []string{} }, + CacheSizeMax: 1024 * 1024, // 1MB + }) + + key := "alpha" + if err := d.Write(key, []byte{'1', '2', '3'}); err != nil { + panic(err) + } + + value, err := d.Read(key) + if err != nil { + panic(err) + } + fmt.Printf("%v\n", value) + + if err := d.Erase(key); err != nil { + panic(err) + } +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/import_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/import_test.go new file mode 100644 index 00000000000..a08ac7c70cb --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/import_test.go @@ -0,0 +1,76 @@ +package diskv_test + +import ( + "bytes" + "io/ioutil" + "os" + + "github.com/peterbourgon/diskv" + + "testing" +) + +func TestImportMove(t *testing.T) { + b := []byte(`0123456789`) + f, err := ioutil.TempFile("", "temp-test") + if err != nil { + t.Fatal(err) + } + if _, err := f.Write(b); err != nil { + t.Fatal(err) + } + f.Close() + + d := diskv.New(diskv.Options{ + BasePath: "test-import-move", + }) + defer d.EraseAll() + + key := "key" + + if err := d.Write(key, []byte(`TBD`)); err != nil { + t.Fatal(err) + } + + if err := d.Import(f.Name(), key, true); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(f.Name()); err == nil || !os.IsNotExist(err) { + t.Errorf("expected temp file to be gone, but err = %v", err) + } + + if !d.Has(key) { + t.Errorf("%q not present", key) + } + + if buf, err := d.Read(key); err != nil || bytes.Compare(b, buf) != 0 { + t.Errorf("want %q, have %q (err = %v)", string(b), string(buf), err) + } +} + +func TestImportCopy(t *testing.T) { + b := []byte(`¡åéîòü!`) + + f, err := ioutil.TempFile("", "temp-test") + if err != nil { + t.Fatal(err) + } + if _, err := f.Write(b); err != nil { + t.Fatal(err) + } + f.Close() + + d := diskv.New(diskv.Options{ + BasePath: "test-import-copy", + }) + defer d.EraseAll() + + if err := d.Import(f.Name(), "key", false); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(f.Name()); err != nil { + t.Errorf("expected temp file to remain, but got err = %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/index.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/index.go new file mode 100644 index 00000000000..1481b2c763b --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/index.go @@ -0,0 +1,116 @@ +package diskv + +import ( + "sync" + + "github.com/petar/GoLLRB/llrb" +) + +// Index is a generic interface for things that can +// provide an ordered list of keys. +type Index interface { + Initialize(less LessFunction, keys <-chan string) + Insert(key string) + Delete(key string) + Keys(from string, n int) []string +} + +// LessFunction is used to initialize an Index of keys in a specific order. +type LessFunction func(string, string) bool + +// llrbString is a custom data type that satisfies the LLRB Less interface, +// making the strings it wraps sortable by the LLRB package. +type llrbString struct { + s string + l LessFunction +} + +// Less satisfies the llrb.Less interface using the llrbString's LessFunction. +func (s llrbString) Less(i llrb.Item) bool { + return s.l(s.s, i.(llrbString).s) +} + +// LLRBIndex is an implementation of the Index interface +// using Petar Maymounkov's LLRB tree. +type LLRBIndex struct { + sync.RWMutex + LessFunction + *llrb.LLRB +} + +// Initialize populates the LLRB tree with data from the keys channel, +// according to the passed less function. It's destructive to the LLRBIndex. +func (i *LLRBIndex) Initialize(less LessFunction, keys <-chan string) { + i.Lock() + defer i.Unlock() + i.LessFunction = less + i.LLRB = rebuild(less, keys) +} + +// Insert inserts the given key (only) into the LLRB tree. +func (i *LLRBIndex) Insert(key string) { + i.Lock() + defer i.Unlock() + if i.LLRB == nil || i.LessFunction == nil { + panic("uninitialized index") + } + i.LLRB.ReplaceOrInsert(llrbString{s: key, l: i.LessFunction}) +} + +// Delete removes the given key (only) from the LLRB tree. +func (i *LLRBIndex) Delete(key string) { + i.Lock() + defer i.Unlock() + if i.LLRB == nil || i.LessFunction == nil { + panic("uninitialized index") + } + i.LLRB.Delete(llrbString{s: key, l: i.LessFunction}) +} + +// Keys yields a maximum of n keys in order. If the passed 'from' key is empty, +// Keys will return the first n keys. If the passed 'from' key is non-empty, the +// first key in the returned slice will be the key that immediately follows the +// passed key, in key order. +func (i *LLRBIndex) Keys(from string, n int) []string { + i.RLock() + defer i.RUnlock() + + if i.LLRB == nil || i.LessFunction == nil { + panic("uninitialized index") + } + + if i.LLRB.Len() <= 0 { + return []string{} + } + + llrbFrom := llrbString{s: from, l: i.LessFunction} + skipFirst := true + if len(from) <= 0 || !i.LLRB.Has(llrbFrom) { + // no such key, so start at the top + llrbFrom = i.LLRB.Min().(llrbString) + skipFirst = false + } + + keys := []string{} + iterator := func(i llrb.Item) bool { + keys = append(keys, i.(llrbString).s) + return len(keys) < n + } + i.LLRB.AscendGreaterOrEqual(llrbFrom, iterator) + + if skipFirst && len(keys) > 0 { + keys = keys[1:] + } + + return keys +} + +// rebuildIndex does the work of regenerating the index +// with the given keys. +func rebuild(less LessFunction, keys <-chan string) *llrb.LLRB { + tree := llrb.New() + for key := range keys { + tree.ReplaceOrInsert(llrbString{s: key, l: less}) + } + return tree +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/index_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/index_test.go new file mode 100644 index 00000000000..2eed3b8a6b0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/index_test.go @@ -0,0 +1,126 @@ +package diskv + +import ( + "bytes" + "testing" + "time" +) + +func strLess(a, b string) bool { return a < b } + +func cmpStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func (d *Diskv) isIndexed(key string) bool { + if d.Index == nil { + return false + } + + for _, got := range d.Index.Keys("", 1000) { + if got == key { + return true + } + } + return false +} + +func TestIndexOrder(t *testing.T) { + d := New(Options{ + BasePath: "index-test", + Transform: func(string) []string { return []string{} }, + CacheSizeMax: 1024, + Index: &LLRBIndex{}, + IndexLess: strLess, + }) + defer d.EraseAll() + + v := []byte{'1', '2', '3'} + d.Write("a", v) + if !d.isIndexed("a") { + t.Fatalf("'a' not indexed after write") + } + d.Write("1", v) + d.Write("m", v) + d.Write("-", v) + d.Write("A", v) + + expectedKeys := []string{"-", "1", "A", "a", "m"} + keys := []string{} + for _, key := range d.Index.Keys("", 100) { + keys = append(keys, key) + } + + if !cmpStrings(keys, expectedKeys) { + t.Fatalf("got %s, expected %s", keys, expectedKeys) + } +} + +func TestIndexLoad(t *testing.T) { + d1 := New(Options{ + BasePath: "index-test", + Transform: func(string) []string { return []string{} }, + CacheSizeMax: 1024, + }) + defer d1.EraseAll() + + val := []byte{'1', '2', '3'} + keys := []string{"a", "b", "c", "d", "e", "f", "g"} + for _, key := range keys { + d1.Write(key, val) + } + + d2 := New(Options{ + BasePath: "index-test", + Transform: func(string) []string { return []string{} }, + CacheSizeMax: 1024, + Index: &LLRBIndex{}, + IndexLess: strLess, + }) + defer d2.EraseAll() + + // check d2 has properly loaded existing d1 data + for _, key := range keys { + if !d2.isIndexed(key) { + t.Fatalf("key '%s' not indexed on secondary", key) + } + } + + // cache one + if readValue, err := d2.Read(keys[0]); err != nil { + t.Fatalf("%s", err) + } else if bytes.Compare(val, readValue) != 0 { + t.Fatalf("%s: got %s, expected %s", keys[0], readValue, val) + } + + // make sure it got cached + for i := 0; i < 10 && !d2.isCached(keys[0]); i++ { + time.Sleep(10 * time.Millisecond) + } + if !d2.isCached(keys[0]) { + t.Fatalf("key '%s' not cached", keys[0]) + } + + // kill the disk + d1.EraseAll() + + // cached value should still be there in the second + if readValue, err := d2.Read(keys[0]); err != nil { + t.Fatalf("%s", err) + } else if bytes.Compare(val, readValue) != 0 { + t.Fatalf("%s: got %s, expected %s", keys[0], readValue, val) + } + + // but not in the original + if _, err := d1.Read(keys[0]); err == nil { + t.Fatalf("expected error reading from flushed store") + } +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/issues_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/issues_test.go new file mode 100644 index 00000000000..0b0b1090823 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/issues_test.go @@ -0,0 +1,121 @@ +package diskv + +import ( + "bytes" + "io/ioutil" + "sync" + "testing" + "time" +) + +// ReadStream from cache shouldn't panic on a nil dereference from a nonexistent +// Compression :) +func TestIssue2A(t *testing.T) { + d := New(Options{ + BasePath: "test-issue-2a", + Transform: func(string) []string { return []string{} }, + CacheSizeMax: 1024, + }) + defer d.EraseAll() + + input := "abcdefghijklmnopqrstuvwxy" + key, writeBuf, sync := "a", bytes.NewBufferString(input), false + if err := d.WriteStream(key, writeBuf, sync); err != nil { + t.Fatal(err) + } + + for i := 0; i < 2; i++ { + began := time.Now() + rc, err := d.ReadStream(key, false) + if err != nil { + t.Fatal(err) + } + buf, err := ioutil.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + if !cmpBytes(buf, []byte(input)) { + t.Fatalf("read #%d: '%s' != '%s'", i+1, string(buf), input) + } + rc.Close() + t.Logf("read #%d in %s", i+1, time.Since(began)) + } +} + +// ReadStream on a key that resolves to a directory should return an error. +func TestIssue2B(t *testing.T) { + blockTransform := func(s string) []string { + transformBlockSize := 3 + sliceSize := len(s) / transformBlockSize + pathSlice := make([]string, sliceSize) + for i := 0; i < sliceSize; i++ { + from, to := i*transformBlockSize, (i*transformBlockSize)+transformBlockSize + pathSlice[i] = s[from:to] + } + return pathSlice + } + + d := New(Options{ + BasePath: "test-issue-2b", + Transform: blockTransform, + CacheSizeMax: 0, + }) + defer d.EraseAll() + + v := []byte{'1', '2', '3'} + if err := d.Write("abcabc", v); err != nil { + t.Fatal(err) + } + + _, err := d.ReadStream("abc", false) + if err == nil { + t.Fatal("ReadStream('abc') should return error") + } + t.Logf("ReadStream('abc') returned error: %v", err) +} + +// Ensure ReadStream with direct=true isn't racy. +func TestIssue17(t *testing.T) { + var ( + basePath = "test-data" + ) + + dWrite := New(Options{ + BasePath: basePath, + CacheSizeMax: 0, + }) + defer dWrite.EraseAll() + + dRead := New(Options{ + BasePath: basePath, + CacheSizeMax: 50, + }) + + cases := map[string]string{ + "a": `1234567890`, + "b": `2345678901`, + "c": `3456789012`, + "d": `4567890123`, + "e": `5678901234`, + } + + for k, v := range cases { + if err := dWrite.Write(k, []byte(v)); err != nil { + t.Fatalf("during write: %s", err) + } + dRead.Read(k) // ensure it's added to cache + } + + var wg sync.WaitGroup + start := make(chan struct{}) + for k, v := range cases { + wg.Add(1) + go func(k, v string) { + <-start + dRead.ReadStream(k, true) + wg.Done() + }(k, v) + } + close(start) + wg.Wait() +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/keys_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/keys_test.go new file mode 100644 index 00000000000..222e1c44414 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/keys_test.go @@ -0,0 +1,231 @@ +package diskv_test + +import ( + "reflect" + "runtime" + "strings" + "testing" + + "github.com/peterbourgon/diskv" +) + +var ( + keysTestData = map[string]string{ + "ab01cd01": "When we started building CoreOS", + "ab01cd02": "we looked at all the various components available to us", + "ab01cd03": "re-using the best tools", + "ef01gh04": "and building the ones that did not exist", + "ef02gh05": "We believe strongly in the Unix philosophy", + "xxxxxxxx": "tools should be independently useful", + } + + prefixes = []string{ + "", // all + "a", + "ab", + "ab0", + "ab01", + "ab01cd0", + "ab01cd01", + "ab01cd01x", // none + "b", // none + "b0", // none + "0", // none + "01", // none + "e", + "ef", + "efx", // none + "ef01gh0", + "ef01gh04", + "ef01gh05", + "ef01gh06", // none + } +) + +func TestKeysFlat(t *testing.T) { + transform := func(s string) []string { + if s == "" { + t.Fatalf(`transform should not be called with ""`) + } + return []string{} + } + d := diskv.New(diskv.Options{ + BasePath: "test-data", + Transform: transform, + }) + defer d.EraseAll() + + for k, v := range keysTestData { + d.Write(k, []byte(v)) + } + + checkKeys(t, d.Keys(nil), keysTestData) +} + +func TestKeysNested(t *testing.T) { + d := diskv.New(diskv.Options{ + BasePath: "test-data", + Transform: blockTransform(2), + }) + defer d.EraseAll() + + for k, v := range keysTestData { + d.Write(k, []byte(v)) + } + + checkKeys(t, d.Keys(nil), keysTestData) +} + +func TestKeysPrefixFlat(t *testing.T) { + d := diskv.New(diskv.Options{ + BasePath: "test-data", + }) + defer d.EraseAll() + + for k, v := range keysTestData { + d.Write(k, []byte(v)) + } + + for _, prefix := range prefixes { + checkKeys(t, d.KeysPrefix(prefix, nil), filterPrefix(keysTestData, prefix)) + } +} + +func TestKeysPrefixNested(t *testing.T) { + d := diskv.New(diskv.Options{ + BasePath: "test-data", + Transform: blockTransform(2), + }) + defer d.EraseAll() + + for k, v := range keysTestData { + d.Write(k, []byte(v)) + } + + for _, prefix := range prefixes { + checkKeys(t, d.KeysPrefix(prefix, nil), filterPrefix(keysTestData, prefix)) + } +} + +func TestKeysCancel(t *testing.T) { + d := diskv.New(diskv.Options{ + BasePath: "test-data", + }) + defer d.EraseAll() + + for k, v := range keysTestData { + d.Write(k, []byte(v)) + } + + var ( + cancel = make(chan struct{}) + received = 0 + cancelAfter = len(keysTestData) / 2 + ) + + for key := range d.Keys(cancel) { + received++ + + if received >= cancelAfter { + close(cancel) + runtime.Gosched() // allow walker to detect cancel + } + + t.Logf("received %d: %q", received, key) + } + + if want, have := cancelAfter, received; want != have { + t.Errorf("want %d, have %d") + } +} + +func checkKeys(t *testing.T, c <-chan string, want map[string]string) { + for k := range c { + if _, ok := want[k]; !ok { + t.Errorf("%q yielded but not expected", k) + continue + } + + delete(want, k) + t.Logf("%q yielded OK", k) + } + + if len(want) != 0 { + t.Errorf("%d expected key(s) not yielded: %s", len(want), strings.Join(flattenKeys(want), ", ")) + } +} + +func blockTransform(blockSize int) func(string) []string { + return func(s string) []string { + var ( + sliceSize = len(s) / blockSize + pathSlice = make([]string, sliceSize) + ) + for i := 0; i < sliceSize; i++ { + from, to := i*blockSize, (i*blockSize)+blockSize + pathSlice[i] = s[from:to] + } + return pathSlice + } +} + +func filterPrefix(in map[string]string, prefix string) map[string]string { + out := map[string]string{} + for k, v := range in { + if strings.HasPrefix(k, prefix) { + out[k] = v + } + } + return out +} + +func TestFilterPrefix(t *testing.T) { + input := map[string]string{ + "all": "", + "and": "", + "at": "", + "available": "", + "best": "", + "building": "", + "components": "", + "coreos": "", + "did": "", + "exist": "", + "looked": "", + "not": "", + "ones": "", + "re-using": "", + "started": "", + "that": "", + "the": "", + "to": "", + "tools": "", + "us": "", + "various": "", + "we": "", + "when": "", + } + + for prefix, want := range map[string]map[string]string{ + "a": map[string]string{"all": "", "and": "", "at": "", "available": ""}, + "al": map[string]string{"all": ""}, + "all": map[string]string{"all": ""}, + "alll": map[string]string{}, + "c": map[string]string{"components": "", "coreos": ""}, + "co": map[string]string{"components": "", "coreos": ""}, + "com": map[string]string{"components": ""}, + } { + have := filterPrefix(input, prefix) + if !reflect.DeepEqual(want, have) { + t.Errorf("%q: want %v, have %v", prefix, flattenKeys(want), flattenKeys(have)) + } + } +} + +func flattenKeys(m map[string]string) []string { + a := make([]string, 0, len(m)) + for k := range m { + a = append(a, k) + } + return a +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/speed_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/speed_test.go new file mode 100644 index 00000000000..43837c0c489 --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/speed_test.go @@ -0,0 +1,153 @@ +package diskv + +import ( + "fmt" + "math/rand" + "testing" +) + +func shuffle(keys []string) { + ints := rand.Perm(len(keys)) + for i := range keys { + keys[i], keys[ints[i]] = keys[ints[i]], keys[i] + } +} + +func genValue(size int) []byte { + v := make([]byte, size) + for i := 0; i < size; i++ { + v[i] = uint8((rand.Int() % 26) + 97) // a-z + } + return v +} + +const ( + keyCount = 1000 +) + +func genKeys() []string { + keys := make([]string, keyCount) + for i := 0; i < keyCount; i++ { + keys[i] = fmt.Sprintf("%d", i) + } + return keys +} + +func (d *Diskv) load(keys []string, val []byte) { + for _, key := range keys { + d.Write(key, val) + } +} + +func benchRead(b *testing.B, size, cachesz int) { + b.StopTimer() + d := New(Options{ + BasePath: "speed-test", + Transform: func(string) []string { return []string{} }, + CacheSizeMax: uint64(cachesz), + }) + defer d.EraseAll() + + keys := genKeys() + value := genValue(size) + d.load(keys, value) + shuffle(keys) + b.SetBytes(int64(size)) + + b.StartTimer() + for i := 0; i < b.N; i++ { + _, _ = d.Read(keys[i%len(keys)]) + } + b.StopTimer() +} + +func benchWrite(b *testing.B, size int, withIndex bool) { + b.StopTimer() + + options := Options{ + BasePath: "speed-test", + Transform: func(string) []string { return []string{} }, + CacheSizeMax: 0, + } + if withIndex { + options.Index = &LLRBIndex{} + options.IndexLess = strLess + } + + d := New(options) + defer d.EraseAll() + keys := genKeys() + value := genValue(size) + shuffle(keys) + b.SetBytes(int64(size)) + + b.StartTimer() + for i := 0; i < b.N; i++ { + d.Write(keys[i%len(keys)], value) + } + b.StopTimer() +} + +func BenchmarkWrite__32B_NoIndex(b *testing.B) { + benchWrite(b, 32, false) +} + +func BenchmarkWrite__1KB_NoIndex(b *testing.B) { + benchWrite(b, 1024, false) +} + +func BenchmarkWrite__4KB_NoIndex(b *testing.B) { + benchWrite(b, 4096, false) +} + +func BenchmarkWrite_10KB_NoIndex(b *testing.B) { + benchWrite(b, 10240, false) +} + +func BenchmarkWrite__32B_WithIndex(b *testing.B) { + benchWrite(b, 32, true) +} + +func BenchmarkWrite__1KB_WithIndex(b *testing.B) { + benchWrite(b, 1024, true) +} + +func BenchmarkWrite__4KB_WithIndex(b *testing.B) { + benchWrite(b, 4096, true) +} + +func BenchmarkWrite_10KB_WithIndex(b *testing.B) { + benchWrite(b, 10240, true) +} + +func BenchmarkRead__32B_NoCache(b *testing.B) { + benchRead(b, 32, 0) +} + +func BenchmarkRead__1KB_NoCache(b *testing.B) { + benchRead(b, 1024, 0) +} + +func BenchmarkRead__4KB_NoCache(b *testing.B) { + benchRead(b, 4096, 0) +} + +func BenchmarkRead_10KB_NoCache(b *testing.B) { + benchRead(b, 10240, 0) +} + +func BenchmarkRead__32B_WithCache(b *testing.B) { + benchRead(b, 32, keyCount*32*2) +} + +func BenchmarkRead__1KB_WithCache(b *testing.B) { + benchRead(b, 1024, keyCount*1024*2) +} + +func BenchmarkRead__4KB_WithCache(b *testing.B) { + benchRead(b, 4096, keyCount*4096*2) +} + +func BenchmarkRead_10KB_WithCache(b *testing.B) { + benchRead(b, 10240, keyCount*4096*2) +} diff --git a/Godeps/_workspace/src/github.com/peterbourgon/diskv/stream_test.go b/Godeps/_workspace/src/github.com/peterbourgon/diskv/stream_test.go new file mode 100644 index 00000000000..7991dbff28d --- /dev/null +++ b/Godeps/_workspace/src/github.com/peterbourgon/diskv/stream_test.go @@ -0,0 +1,117 @@ +package diskv + +import ( + "bytes" + "io/ioutil" + "testing" +) + +func TestBasicStreamCaching(t *testing.T) { + d := New(Options{ + BasePath: "test-data", + CacheSizeMax: 1024, + }) + defer d.EraseAll() + + input := "a1b2c3" + key, writeBuf, sync := "a", bytes.NewBufferString(input), true + if err := d.WriteStream(key, writeBuf, sync); err != nil { + t.Fatal(err) + } + + if d.isCached(key) { + t.Fatalf("'%s' cached, but shouldn't be (yet)", key) + } + + rc, err := d.ReadStream(key, false) + if err != nil { + t.Fatal(err) + } + + readBuf, err := ioutil.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + + if !cmpBytes(readBuf, []byte(input)) { + t.Fatalf("'%s' != '%s'", string(readBuf), input) + } + + if !d.isCached(key) { + t.Fatalf("'%s' isn't cached, but should be", key) + } +} + +func TestReadStreamDirect(t *testing.T) { + var ( + basePath = "test-data" + ) + dWrite := New(Options{ + BasePath: basePath, + CacheSizeMax: 0, + }) + defer dWrite.EraseAll() + dRead := New(Options{ + BasePath: basePath, + CacheSizeMax: 1024, + }) + + // Write + key, val1, val2 := "a", []byte(`1234567890`), []byte(`aaaaaaaaaa`) + if err := dWrite.Write(key, val1); err != nil { + t.Fatalf("during first write: %s", err) + } + + // First, caching read. + val, err := dRead.Read(key) + if err != nil { + t.Fatalf("during initial read: %s", err) + } + t.Logf("read 1: %s => %s", key, string(val)) + if !cmpBytes(val1, val) { + t.Errorf("expected %q, got %q", string(val1), string(val)) + } + if !dRead.isCached(key) { + t.Errorf("%q should be cached, but isn't", key) + } + + // Write a different value. + if err := dWrite.Write(key, val2); err != nil { + t.Fatalf("during second write: %s", err) + } + + // Second read, should hit cache and get the old value. + val, err = dRead.Read(key) + if err != nil { + t.Fatalf("during second (cache-hit) read: %s", err) + } + t.Logf("read 2: %s => %s", key, string(val)) + if !cmpBytes(val1, val) { + t.Errorf("expected %q, got %q", string(val1), string(val)) + } + + // Third, direct read, should get the updated value. + rc, err := dRead.ReadStream(key, true) + if err != nil { + t.Fatalf("during third (direct) read, ReadStream: %s", err) + } + defer rc.Close() + val, err = ioutil.ReadAll(rc) + if err != nil { + t.Fatalf("during third (direct) read, ReadAll: %s", err) + } + t.Logf("read 3: %s => %s", key, string(val)) + if !cmpBytes(val2, val) { + t.Errorf("expected %q, got %q", string(val1), string(val)) + } + + // Fourth read, should hit cache and get the new value. + val, err = dRead.Read(key) + if err != nil { + t.Fatalf("during fourth (cache-hit) read: %s", err) + } + t.Logf("read 4: %s => %s", key, string(val)) + if !cmpBytes(val2, val) { + t.Errorf("expected %q, got %q", string(val1), string(val)) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/cast5/cast5.go b/Godeps/_workspace/src/golang.org/x/crypto/cast5/cast5.go new file mode 100644 index 00000000000..8c1b299bf2f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/cast5/cast5.go @@ -0,0 +1,526 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cast5 implements CAST5, as defined in RFC 2144. CAST5 is a common +// OpenPGP cipher. +package cast5 + +import "errors" + +const BlockSize = 8 +const KeySize = 16 + +type Cipher struct { + masking [16]uint32 + rotate [16]uint8 +} + +func NewCipher(key []byte) (c *Cipher, err error) { + if len(key) != KeySize { + return nil, errors.New("CAST5: keys must be 16 bytes") + } + + c = new(Cipher) + c.keySchedule(key) + return +} + +func (c *Cipher) BlockSize() int { + return BlockSize +} + +func (c *Cipher) Encrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +func (c *Cipher) Decrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +type keyScheduleA [4][7]uint8 +type keyScheduleB [4][5]uint8 + +// keyScheduleRound contains the magic values for a round of the key schedule. +// The keyScheduleA deals with the lines like: +// z0z1z2z3 = x0x1x2x3 ^ S5[xD] ^ S6[xF] ^ S7[xC] ^ S8[xE] ^ S7[x8] +// Conceptually, both x and z are in the same array, x first. The first +// element describes which word of this array gets written to and the +// second, which word gets read. So, for the line above, it's "4, 0", because +// it's writing to the first word of z, which, being after x, is word 4, and +// reading from the first word of x: word 0. +// +// Next are the indexes into the S-boxes. Now the array is treated as bytes. So +// "xD" is 0xd. The first byte of z is written as "16 + 0", just to be clear +// that it's z that we're indexing. +// +// keyScheduleB deals with lines like: +// K1 = S5[z8] ^ S6[z9] ^ S7[z7] ^ S8[z6] ^ S5[z2] +// "K1" is ignored because key words are always written in order. So the five +// elements are the S-box indexes. They use the same form as in keyScheduleA, +// above. + +type keyScheduleRound struct{} +type keySchedule []keyScheduleRound + +var schedule = []struct { + a keyScheduleA + b keyScheduleB +}{ + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 0x8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 8, 16 + 9, 16 + 7, 16 + 6, 16 + 2}, + {16 + 0xa, 16 + 0xb, 16 + 5, 16 + 4, 16 + 6}, + {16 + 0xc, 16 + 0xd, 16 + 3, 16 + 2, 16 + 9}, + {16 + 0xe, 16 + 0xf, 16 + 1, 16 + 0, 16 + 0xc}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {3, 2, 0xc, 0xd, 8}, + {1, 0, 0xe, 0xf, 0xd}, + {7, 6, 8, 9, 3}, + {5, 4, 0xa, 0xb, 7}, + }, + }, + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 3, 16 + 2, 16 + 0xc, 16 + 0xd, 16 + 9}, + {16 + 1, 16 + 0, 16 + 0xe, 16 + 0xf, 16 + 0xc}, + {16 + 7, 16 + 6, 16 + 8, 16 + 9, 16 + 2}, + {16 + 5, 16 + 4, 16 + 0xa, 16 + 0xb, 16 + 6}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {8, 9, 7, 6, 3}, + {0xa, 0xb, 5, 4, 7}, + {0xc, 0xd, 3, 2, 8}, + {0xe, 0xf, 1, 0, 0xd}, + }, + }, +} + +func (c *Cipher) keySchedule(in []byte) { + var t [8]uint32 + var k [32]uint32 + + for i := 0; i < 4; i++ { + j := i * 4 + t[i] = uint32(in[j])<<24 | uint32(in[j+1])<<16 | uint32(in[j+2])<<8 | uint32(in[j+3]) + } + + x := []byte{6, 7, 4, 5} + ki := 0 + + for half := 0; half < 2; half++ { + for _, round := range schedule { + for j := 0; j < 4; j++ { + var a [7]uint8 + copy(a[:], round.a[j][:]) + w := t[a[1]] + w ^= sBox[4][(t[a[2]>>2]>>(24-8*(a[2]&3)))&0xff] + w ^= sBox[5][(t[a[3]>>2]>>(24-8*(a[3]&3)))&0xff] + w ^= sBox[6][(t[a[4]>>2]>>(24-8*(a[4]&3)))&0xff] + w ^= sBox[7][(t[a[5]>>2]>>(24-8*(a[5]&3)))&0xff] + w ^= sBox[x[j]][(t[a[6]>>2]>>(24-8*(a[6]&3)))&0xff] + t[a[0]] = w + } + + for j := 0; j < 4; j++ { + var b [5]uint8 + copy(b[:], round.b[j][:]) + w := sBox[4][(t[b[0]>>2]>>(24-8*(b[0]&3)))&0xff] + w ^= sBox[5][(t[b[1]>>2]>>(24-8*(b[1]&3)))&0xff] + w ^= sBox[6][(t[b[2]>>2]>>(24-8*(b[2]&3)))&0xff] + w ^= sBox[7][(t[b[3]>>2]>>(24-8*(b[3]&3)))&0xff] + w ^= sBox[4+j][(t[b[4]>>2]>>(24-8*(b[4]&3)))&0xff] + k[ki] = w + ki++ + } + } + } + + for i := 0; i < 16; i++ { + c.masking[i] = k[i] + c.rotate[i] = uint8(k[16+i] & 0x1f) + } +} + +// These are the three 'f' functions. See RFC 2144, section 2.2. +func f1(d, m uint32, r uint8) uint32 { + t := m + d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] ^ sBox[1][(I>>16)&0xff]) - sBox[2][(I>>8)&0xff]) + sBox[3][I&0xff] +} + +func f2(d, m uint32, r uint8) uint32 { + t := m ^ d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] - sBox[1][(I>>16)&0xff]) + sBox[2][(I>>8)&0xff]) ^ sBox[3][I&0xff] +} + +func f3(d, m uint32, r uint8) uint32 { + t := m - d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] + sBox[1][(I>>16)&0xff]) ^ sBox[2][(I>>8)&0xff]) - sBox[3][I&0xff] +} + +var sBox = [8][256]uint32{ + { + 0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f, 0x9c004dd3, 0x6003e540, 0xcf9fc949, + 0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0, 0x15c361d2, 0xc2e7661d, 0x22d4ff8e, + 0x28683b6f, 0xc07fd059, 0xff2379c8, 0x775f50e2, 0x43c340d3, 0xdf2f8656, 0x887ca41a, 0xa2d2bd2d, + 0xa1c9e0d6, 0x346c4819, 0x61b76d87, 0x22540f2f, 0x2abe32e1, 0xaa54166b, 0x22568e3a, 0xa2d341d0, + 0x66db40c8, 0xa784392f, 0x004dff2f, 0x2db9d2de, 0x97943fac, 0x4a97c1d8, 0x527644b7, 0xb5f437a7, + 0xb82cbaef, 0xd751d159, 0x6ff7f0ed, 0x5a097a1f, 0x827b68d0, 0x90ecf52e, 0x22b0c054, 0xbc8e5935, + 0x4b6d2f7f, 0x50bb64a2, 0xd2664910, 0xbee5812d, 0xb7332290, 0xe93b159f, 0xb48ee411, 0x4bff345d, + 0xfd45c240, 0xad31973f, 0xc4f6d02e, 0x55fc8165, 0xd5b1caad, 0xa1ac2dae, 0xa2d4b76d, 0xc19b0c50, + 0x882240f2, 0x0c6e4f38, 0xa4e4bfd7, 0x4f5ba272, 0x564c1d2f, 0xc59c5319, 0xb949e354, 0xb04669fe, + 0xb1b6ab8a, 0xc71358dd, 0x6385c545, 0x110f935d, 0x57538ad5, 0x6a390493, 0xe63d37e0, 0x2a54f6b3, + 0x3a787d5f, 0x6276a0b5, 0x19a6fcdf, 0x7a42206a, 0x29f9d4d5, 0xf61b1891, 0xbb72275e, 0xaa508167, + 0x38901091, 0xc6b505eb, 0x84c7cb8c, 0x2ad75a0f, 0x874a1427, 0xa2d1936b, 0x2ad286af, 0xaa56d291, + 0xd7894360, 0x425c750d, 0x93b39e26, 0x187184c9, 0x6c00b32d, 0x73e2bb14, 0xa0bebc3c, 0x54623779, + 0x64459eab, 0x3f328b82, 0x7718cf82, 0x59a2cea6, 0x04ee002e, 0x89fe78e6, 0x3fab0950, 0x325ff6c2, + 0x81383f05, 0x6963c5c8, 0x76cb5ad6, 0xd49974c9, 0xca180dcf, 0x380782d5, 0xc7fa5cf6, 0x8ac31511, + 0x35e79e13, 0x47da91d0, 0xf40f9086, 0xa7e2419e, 0x31366241, 0x051ef495, 0xaa573b04, 0x4a805d8d, + 0x548300d0, 0x00322a3c, 0xbf64cddf, 0xba57a68e, 0x75c6372b, 0x50afd341, 0xa7c13275, 0x915a0bf5, + 0x6b54bfab, 0x2b0b1426, 0xab4cc9d7, 0x449ccd82, 0xf7fbf265, 0xab85c5f3, 0x1b55db94, 0xaad4e324, + 0xcfa4bd3f, 0x2deaa3e2, 0x9e204d02, 0xc8bd25ac, 0xeadf55b3, 0xd5bd9e98, 0xe31231b2, 0x2ad5ad6c, + 0x954329de, 0xadbe4528, 0xd8710f69, 0xaa51c90f, 0xaa786bf6, 0x22513f1e, 0xaa51a79b, 0x2ad344cc, + 0x7b5a41f0, 0xd37cfbad, 0x1b069505, 0x41ece491, 0xb4c332e6, 0x032268d4, 0xc9600acc, 0xce387e6d, + 0xbf6bb16c, 0x6a70fb78, 0x0d03d9c9, 0xd4df39de, 0xe01063da, 0x4736f464, 0x5ad328d8, 0xb347cc96, + 0x75bb0fc3, 0x98511bfb, 0x4ffbcc35, 0xb58bcf6a, 0xe11f0abc, 0xbfc5fe4a, 0xa70aec10, 0xac39570a, + 0x3f04442f, 0x6188b153, 0xe0397a2e, 0x5727cb79, 0x9ceb418f, 0x1cacd68d, 0x2ad37c96, 0x0175cb9d, + 0xc69dff09, 0xc75b65f0, 0xd9db40d8, 0xec0e7779, 0x4744ead4, 0xb11c3274, 0xdd24cb9e, 0x7e1c54bd, + 0xf01144f9, 0xd2240eb1, 0x9675b3fd, 0xa3ac3755, 0xd47c27af, 0x51c85f4d, 0x56907596, 0xa5bb15e6, + 0x580304f0, 0xca042cf1, 0x011a37ea, 0x8dbfaadb, 0x35ba3e4a, 0x3526ffa0, 0xc37b4d09, 0xbc306ed9, + 0x98a52666, 0x5648f725, 0xff5e569d, 0x0ced63d0, 0x7c63b2cf, 0x700b45e1, 0xd5ea50f1, 0x85a92872, + 0xaf1fbda7, 0xd4234870, 0xa7870bf3, 0x2d3b4d79, 0x42e04198, 0x0cd0ede7, 0x26470db8, 0xf881814c, + 0x474d6ad7, 0x7c0c5e5c, 0xd1231959, 0x381b7298, 0xf5d2f4db, 0xab838653, 0x6e2f1e23, 0x83719c9e, + 0xbd91e046, 0x9a56456e, 0xdc39200c, 0x20c8c571, 0x962bda1c, 0xe1e696ff, 0xb141ab08, 0x7cca89b9, + 0x1a69e783, 0x02cc4843, 0xa2f7c579, 0x429ef47d, 0x427b169c, 0x5ac9f049, 0xdd8f0f00, 0x5c8165bf, + }, + { + 0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a, 0xeec5207a, 0x55889c94, 0x72fc0651, + 0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef, 0x5f0c0794, 0x18dcdb7d, 0xa1d6eff3, + 0xa0b52f7b, 0x59e83605, 0xee15b094, 0xe9ffd909, 0xdc440086, 0xef944459, 0xba83ccb3, 0xe0c3cdfb, + 0xd1da4181, 0x3b092ab1, 0xf997f1c1, 0xa5e6cf7b, 0x01420ddb, 0xe4e7ef5b, 0x25a1ff41, 0xe180f806, + 0x1fc41080, 0x179bee7a, 0xd37ac6a9, 0xfe5830a4, 0x98de8b7f, 0x77e83f4e, 0x79929269, 0x24fa9f7b, + 0xe113c85b, 0xacc40083, 0xd7503525, 0xf7ea615f, 0x62143154, 0x0d554b63, 0x5d681121, 0xc866c359, + 0x3d63cf73, 0xcee234c0, 0xd4d87e87, 0x5c672b21, 0x071f6181, 0x39f7627f, 0x361e3084, 0xe4eb573b, + 0x602f64a4, 0xd63acd9c, 0x1bbc4635, 0x9e81032d, 0x2701f50c, 0x99847ab4, 0xa0e3df79, 0xba6cf38c, + 0x10843094, 0x2537a95e, 0xf46f6ffe, 0xa1ff3b1f, 0x208cfb6a, 0x8f458c74, 0xd9e0a227, 0x4ec73a34, + 0xfc884f69, 0x3e4de8df, 0xef0e0088, 0x3559648d, 0x8a45388c, 0x1d804366, 0x721d9bfd, 0xa58684bb, + 0xe8256333, 0x844e8212, 0x128d8098, 0xfed33fb4, 0xce280ae1, 0x27e19ba5, 0xd5a6c252, 0xe49754bd, + 0xc5d655dd, 0xeb667064, 0x77840b4d, 0xa1b6a801, 0x84db26a9, 0xe0b56714, 0x21f043b7, 0xe5d05860, + 0x54f03084, 0x066ff472, 0xa31aa153, 0xdadc4755, 0xb5625dbf, 0x68561be6, 0x83ca6b94, 0x2d6ed23b, + 0xeccf01db, 0xa6d3d0ba, 0xb6803d5c, 0xaf77a709, 0x33b4a34c, 0x397bc8d6, 0x5ee22b95, 0x5f0e5304, + 0x81ed6f61, 0x20e74364, 0xb45e1378, 0xde18639b, 0x881ca122, 0xb96726d1, 0x8049a7e8, 0x22b7da7b, + 0x5e552d25, 0x5272d237, 0x79d2951c, 0xc60d894c, 0x488cb402, 0x1ba4fe5b, 0xa4b09f6b, 0x1ca815cf, + 0xa20c3005, 0x8871df63, 0xb9de2fcb, 0x0cc6c9e9, 0x0beeff53, 0xe3214517, 0xb4542835, 0x9f63293c, + 0xee41e729, 0x6e1d2d7c, 0x50045286, 0x1e6685f3, 0xf33401c6, 0x30a22c95, 0x31a70850, 0x60930f13, + 0x73f98417, 0xa1269859, 0xec645c44, 0x52c877a9, 0xcdff33a6, 0xa02b1741, 0x7cbad9a2, 0x2180036f, + 0x50d99c08, 0xcb3f4861, 0xc26bd765, 0x64a3f6ab, 0x80342676, 0x25a75e7b, 0xe4e6d1fc, 0x20c710e6, + 0xcdf0b680, 0x17844d3b, 0x31eef84d, 0x7e0824e4, 0x2ccb49eb, 0x846a3bae, 0x8ff77888, 0xee5d60f6, + 0x7af75673, 0x2fdd5cdb, 0xa11631c1, 0x30f66f43, 0xb3faec54, 0x157fd7fa, 0xef8579cc, 0xd152de58, + 0xdb2ffd5e, 0x8f32ce19, 0x306af97a, 0x02f03ef8, 0x99319ad5, 0xc242fa0f, 0xa7e3ebb0, 0xc68e4906, + 0xb8da230c, 0x80823028, 0xdcdef3c8, 0xd35fb171, 0x088a1bc8, 0xbec0c560, 0x61a3c9e8, 0xbca8f54d, + 0xc72feffa, 0x22822e99, 0x82c570b4, 0xd8d94e89, 0x8b1c34bc, 0x301e16e6, 0x273be979, 0xb0ffeaa6, + 0x61d9b8c6, 0x00b24869, 0xb7ffce3f, 0x08dc283b, 0x43daf65a, 0xf7e19798, 0x7619b72f, 0x8f1c9ba4, + 0xdc8637a0, 0x16a7d3b1, 0x9fc393b7, 0xa7136eeb, 0xc6bcc63e, 0x1a513742, 0xef6828bc, 0x520365d6, + 0x2d6a77ab, 0x3527ed4b, 0x821fd216, 0x095c6e2e, 0xdb92f2fb, 0x5eea29cb, 0x145892f5, 0x91584f7f, + 0x5483697b, 0x2667a8cc, 0x85196048, 0x8c4bacea, 0x833860d4, 0x0d23e0f9, 0x6c387e8a, 0x0ae6d249, + 0xb284600c, 0xd835731d, 0xdcb1c647, 0xac4c56ea, 0x3ebd81b3, 0x230eabb0, 0x6438bc87, 0xf0b5b1fa, + 0x8f5ea2b3, 0xfc184642, 0x0a036b7a, 0x4fb089bd, 0x649da589, 0xa345415e, 0x5c038323, 0x3e5d3bb9, + 0x43d79572, 0x7e6dd07c, 0x06dfdf1e, 0x6c6cc4ef, 0x7160a539, 0x73bfbe70, 0x83877605, 0x4523ecf1, + }, + { + 0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff, 0x369fe44b, 0x8c1fc644, 0xaececa90, + 0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806, 0xf0ad0548, 0xe13c8d83, 0x927010d5, + 0x11107d9f, 0x07647db9, 0xb2e3e4d4, 0x3d4f285e, 0xb9afa820, 0xfade82e0, 0xa067268b, 0x8272792e, + 0x553fb2c0, 0x489ae22b, 0xd4ef9794, 0x125e3fbc, 0x21fffcee, 0x825b1bfd, 0x9255c5ed, 0x1257a240, + 0x4e1a8302, 0xbae07fff, 0x528246e7, 0x8e57140e, 0x3373f7bf, 0x8c9f8188, 0xa6fc4ee8, 0xc982b5a5, + 0xa8c01db7, 0x579fc264, 0x67094f31, 0xf2bd3f5f, 0x40fff7c1, 0x1fb78dfc, 0x8e6bd2c1, 0x437be59b, + 0x99b03dbf, 0xb5dbc64b, 0x638dc0e6, 0x55819d99, 0xa197c81c, 0x4a012d6e, 0xc5884a28, 0xccc36f71, + 0xb843c213, 0x6c0743f1, 0x8309893c, 0x0feddd5f, 0x2f7fe850, 0xd7c07f7e, 0x02507fbf, 0x5afb9a04, + 0xa747d2d0, 0x1651192e, 0xaf70bf3e, 0x58c31380, 0x5f98302e, 0x727cc3c4, 0x0a0fb402, 0x0f7fef82, + 0x8c96fdad, 0x5d2c2aae, 0x8ee99a49, 0x50da88b8, 0x8427f4a0, 0x1eac5790, 0x796fb449, 0x8252dc15, + 0xefbd7d9b, 0xa672597d, 0xada840d8, 0x45f54504, 0xfa5d7403, 0xe83ec305, 0x4f91751a, 0x925669c2, + 0x23efe941, 0xa903f12e, 0x60270df2, 0x0276e4b6, 0x94fd6574, 0x927985b2, 0x8276dbcb, 0x02778176, + 0xf8af918d, 0x4e48f79e, 0x8f616ddf, 0xe29d840e, 0x842f7d83, 0x340ce5c8, 0x96bbb682, 0x93b4b148, + 0xef303cab, 0x984faf28, 0x779faf9b, 0x92dc560d, 0x224d1e20, 0x8437aa88, 0x7d29dc96, 0x2756d3dc, + 0x8b907cee, 0xb51fd240, 0xe7c07ce3, 0xe566b4a1, 0xc3e9615e, 0x3cf8209d, 0x6094d1e3, 0xcd9ca341, + 0x5c76460e, 0x00ea983b, 0xd4d67881, 0xfd47572c, 0xf76cedd9, 0xbda8229c, 0x127dadaa, 0x438a074e, + 0x1f97c090, 0x081bdb8a, 0x93a07ebe, 0xb938ca15, 0x97b03cff, 0x3dc2c0f8, 0x8d1ab2ec, 0x64380e51, + 0x68cc7bfb, 0xd90f2788, 0x12490181, 0x5de5ffd4, 0xdd7ef86a, 0x76a2e214, 0xb9a40368, 0x925d958f, + 0x4b39fffa, 0xba39aee9, 0xa4ffd30b, 0xfaf7933b, 0x6d498623, 0x193cbcfa, 0x27627545, 0x825cf47a, + 0x61bd8ba0, 0xd11e42d1, 0xcead04f4, 0x127ea392, 0x10428db7, 0x8272a972, 0x9270c4a8, 0x127de50b, + 0x285ba1c8, 0x3c62f44f, 0x35c0eaa5, 0xe805d231, 0x428929fb, 0xb4fcdf82, 0x4fb66a53, 0x0e7dc15b, + 0x1f081fab, 0x108618ae, 0xfcfd086d, 0xf9ff2889, 0x694bcc11, 0x236a5cae, 0x12deca4d, 0x2c3f8cc5, + 0xd2d02dfe, 0xf8ef5896, 0xe4cf52da, 0x95155b67, 0x494a488c, 0xb9b6a80c, 0x5c8f82bc, 0x89d36b45, + 0x3a609437, 0xec00c9a9, 0x44715253, 0x0a874b49, 0xd773bc40, 0x7c34671c, 0x02717ef6, 0x4feb5536, + 0xa2d02fff, 0xd2bf60c4, 0xd43f03c0, 0x50b4ef6d, 0x07478cd1, 0x006e1888, 0xa2e53f55, 0xb9e6d4bc, + 0xa2048016, 0x97573833, 0xd7207d67, 0xde0f8f3d, 0x72f87b33, 0xabcc4f33, 0x7688c55d, 0x7b00a6b0, + 0x947b0001, 0x570075d2, 0xf9bb88f8, 0x8942019e, 0x4264a5ff, 0x856302e0, 0x72dbd92b, 0xee971b69, + 0x6ea22fde, 0x5f08ae2b, 0xaf7a616d, 0xe5c98767, 0xcf1febd2, 0x61efc8c2, 0xf1ac2571, 0xcc8239c2, + 0x67214cb8, 0xb1e583d1, 0xb7dc3e62, 0x7f10bdce, 0xf90a5c38, 0x0ff0443d, 0x606e6dc6, 0x60543a49, + 0x5727c148, 0x2be98a1d, 0x8ab41738, 0x20e1be24, 0xaf96da0f, 0x68458425, 0x99833be5, 0x600d457d, + 0x282f9350, 0x8334b362, 0xd91d1120, 0x2b6d8da0, 0x642b1e31, 0x9c305a00, 0x52bce688, 0x1b03588a, + 0xf7baefd5, 0x4142ed9c, 0xa4315c11, 0x83323ec5, 0xdfef4636, 0xa133c501, 0xe9d3531c, 0xee353783, + }, + { + 0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb, 0x64ad8c57, 0x85510443, 0xfa020ed1, + 0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43, 0x6497b7b1, 0xf3641f63, 0x241e4adf, + 0x28147f5f, 0x4fa2b8cd, 0xc9430040, 0x0cc32220, 0xfdd30b30, 0xc0a5374f, 0x1d2d00d9, 0x24147b15, + 0xee4d111a, 0x0fca5167, 0x71ff904c, 0x2d195ffe, 0x1a05645f, 0x0c13fefe, 0x081b08ca, 0x05170121, + 0x80530100, 0xe83e5efe, 0xac9af4f8, 0x7fe72701, 0xd2b8ee5f, 0x06df4261, 0xbb9e9b8a, 0x7293ea25, + 0xce84ffdf, 0xf5718801, 0x3dd64b04, 0xa26f263b, 0x7ed48400, 0x547eebe6, 0x446d4ca0, 0x6cf3d6f5, + 0x2649abdf, 0xaea0c7f5, 0x36338cc1, 0x503f7e93, 0xd3772061, 0x11b638e1, 0x72500e03, 0xf80eb2bb, + 0xabe0502e, 0xec8d77de, 0x57971e81, 0xe14f6746, 0xc9335400, 0x6920318f, 0x081dbb99, 0xffc304a5, + 0x4d351805, 0x7f3d5ce3, 0xa6c866c6, 0x5d5bcca9, 0xdaec6fea, 0x9f926f91, 0x9f46222f, 0x3991467d, + 0xa5bf6d8e, 0x1143c44f, 0x43958302, 0xd0214eeb, 0x022083b8, 0x3fb6180c, 0x18f8931e, 0x281658e6, + 0x26486e3e, 0x8bd78a70, 0x7477e4c1, 0xb506e07c, 0xf32d0a25, 0x79098b02, 0xe4eabb81, 0x28123b23, + 0x69dead38, 0x1574ca16, 0xdf871b62, 0x211c40b7, 0xa51a9ef9, 0x0014377b, 0x041e8ac8, 0x09114003, + 0xbd59e4d2, 0xe3d156d5, 0x4fe876d5, 0x2f91a340, 0x557be8de, 0x00eae4a7, 0x0ce5c2ec, 0x4db4bba6, + 0xe756bdff, 0xdd3369ac, 0xec17b035, 0x06572327, 0x99afc8b0, 0x56c8c391, 0x6b65811c, 0x5e146119, + 0x6e85cb75, 0xbe07c002, 0xc2325577, 0x893ff4ec, 0x5bbfc92d, 0xd0ec3b25, 0xb7801ab7, 0x8d6d3b24, + 0x20c763ef, 0xc366a5fc, 0x9c382880, 0x0ace3205, 0xaac9548a, 0xeca1d7c7, 0x041afa32, 0x1d16625a, + 0x6701902c, 0x9b757a54, 0x31d477f7, 0x9126b031, 0x36cc6fdb, 0xc70b8b46, 0xd9e66a48, 0x56e55a79, + 0x026a4ceb, 0x52437eff, 0x2f8f76b4, 0x0df980a5, 0x8674cde3, 0xedda04eb, 0x17a9be04, 0x2c18f4df, + 0xb7747f9d, 0xab2af7b4, 0xefc34d20, 0x2e096b7c, 0x1741a254, 0xe5b6a035, 0x213d42f6, 0x2c1c7c26, + 0x61c2f50f, 0x6552daf9, 0xd2c231f8, 0x25130f69, 0xd8167fa2, 0x0418f2c8, 0x001a96a6, 0x0d1526ab, + 0x63315c21, 0x5e0a72ec, 0x49bafefd, 0x187908d9, 0x8d0dbd86, 0x311170a7, 0x3e9b640c, 0xcc3e10d7, + 0xd5cad3b6, 0x0caec388, 0xf73001e1, 0x6c728aff, 0x71eae2a1, 0x1f9af36e, 0xcfcbd12f, 0xc1de8417, + 0xac07be6b, 0xcb44a1d8, 0x8b9b0f56, 0x013988c3, 0xb1c52fca, 0xb4be31cd, 0xd8782806, 0x12a3a4e2, + 0x6f7de532, 0x58fd7eb6, 0xd01ee900, 0x24adffc2, 0xf4990fc5, 0x9711aac5, 0x001d7b95, 0x82e5e7d2, + 0x109873f6, 0x00613096, 0xc32d9521, 0xada121ff, 0x29908415, 0x7fbb977f, 0xaf9eb3db, 0x29c9ed2a, + 0x5ce2a465, 0xa730f32c, 0xd0aa3fe8, 0x8a5cc091, 0xd49e2ce7, 0x0ce454a9, 0xd60acd86, 0x015f1919, + 0x77079103, 0xdea03af6, 0x78a8565e, 0xdee356df, 0x21f05cbe, 0x8b75e387, 0xb3c50651, 0xb8a5c3ef, + 0xd8eeb6d2, 0xe523be77, 0xc2154529, 0x2f69efdf, 0xafe67afb, 0xf470c4b2, 0xf3e0eb5b, 0xd6cc9876, + 0x39e4460c, 0x1fda8538, 0x1987832f, 0xca007367, 0xa99144f8, 0x296b299e, 0x492fc295, 0x9266beab, + 0xb5676e69, 0x9bd3ddda, 0xdf7e052f, 0xdb25701c, 0x1b5e51ee, 0xf65324e6, 0x6afce36c, 0x0316cc04, + 0x8644213e, 0xb7dc59d0, 0x7965291f, 0xccd6fd43, 0x41823979, 0x932bcdf6, 0xb657c34d, 0x4edfd282, + 0x7ae5290c, 0x3cb9536b, 0x851e20fe, 0x9833557e, 0x13ecf0b0, 0xd3ffb372, 0x3f85c5c1, 0x0aef7ed2, + }, + { + 0x7ec90c04, 0x2c6e74b9, 0x9b0e66df, 0xa6337911, 0xb86a7fff, 0x1dd358f5, 0x44dd9d44, 0x1731167f, + 0x08fbf1fa, 0xe7f511cc, 0xd2051b00, 0x735aba00, 0x2ab722d8, 0x386381cb, 0xacf6243a, 0x69befd7a, + 0xe6a2e77f, 0xf0c720cd, 0xc4494816, 0xccf5c180, 0x38851640, 0x15b0a848, 0xe68b18cb, 0x4caadeff, + 0x5f480a01, 0x0412b2aa, 0x259814fc, 0x41d0efe2, 0x4e40b48d, 0x248eb6fb, 0x8dba1cfe, 0x41a99b02, + 0x1a550a04, 0xba8f65cb, 0x7251f4e7, 0x95a51725, 0xc106ecd7, 0x97a5980a, 0xc539b9aa, 0x4d79fe6a, + 0xf2f3f763, 0x68af8040, 0xed0c9e56, 0x11b4958b, 0xe1eb5a88, 0x8709e6b0, 0xd7e07156, 0x4e29fea7, + 0x6366e52d, 0x02d1c000, 0xc4ac8e05, 0x9377f571, 0x0c05372a, 0x578535f2, 0x2261be02, 0xd642a0c9, + 0xdf13a280, 0x74b55bd2, 0x682199c0, 0xd421e5ec, 0x53fb3ce8, 0xc8adedb3, 0x28a87fc9, 0x3d959981, + 0x5c1ff900, 0xfe38d399, 0x0c4eff0b, 0x062407ea, 0xaa2f4fb1, 0x4fb96976, 0x90c79505, 0xb0a8a774, + 0xef55a1ff, 0xe59ca2c2, 0xa6b62d27, 0xe66a4263, 0xdf65001f, 0x0ec50966, 0xdfdd55bc, 0x29de0655, + 0x911e739a, 0x17af8975, 0x32c7911c, 0x89f89468, 0x0d01e980, 0x524755f4, 0x03b63cc9, 0x0cc844b2, + 0xbcf3f0aa, 0x87ac36e9, 0xe53a7426, 0x01b3d82b, 0x1a9e7449, 0x64ee2d7e, 0xcddbb1da, 0x01c94910, + 0xb868bf80, 0x0d26f3fd, 0x9342ede7, 0x04a5c284, 0x636737b6, 0x50f5b616, 0xf24766e3, 0x8eca36c1, + 0x136e05db, 0xfef18391, 0xfb887a37, 0xd6e7f7d4, 0xc7fb7dc9, 0x3063fcdf, 0xb6f589de, 0xec2941da, + 0x26e46695, 0xb7566419, 0xf654efc5, 0xd08d58b7, 0x48925401, 0xc1bacb7f, 0xe5ff550f, 0xb6083049, + 0x5bb5d0e8, 0x87d72e5a, 0xab6a6ee1, 0x223a66ce, 0xc62bf3cd, 0x9e0885f9, 0x68cb3e47, 0x086c010f, + 0xa21de820, 0xd18b69de, 0xf3f65777, 0xfa02c3f6, 0x407edac3, 0xcbb3d550, 0x1793084d, 0xb0d70eba, + 0x0ab378d5, 0xd951fb0c, 0xded7da56, 0x4124bbe4, 0x94ca0b56, 0x0f5755d1, 0xe0e1e56e, 0x6184b5be, + 0x580a249f, 0x94f74bc0, 0xe327888e, 0x9f7b5561, 0xc3dc0280, 0x05687715, 0x646c6bd7, 0x44904db3, + 0x66b4f0a3, 0xc0f1648a, 0x697ed5af, 0x49e92ff6, 0x309e374f, 0x2cb6356a, 0x85808573, 0x4991f840, + 0x76f0ae02, 0x083be84d, 0x28421c9a, 0x44489406, 0x736e4cb8, 0xc1092910, 0x8bc95fc6, 0x7d869cf4, + 0x134f616f, 0x2e77118d, 0xb31b2be1, 0xaa90b472, 0x3ca5d717, 0x7d161bba, 0x9cad9010, 0xaf462ba2, + 0x9fe459d2, 0x45d34559, 0xd9f2da13, 0xdbc65487, 0xf3e4f94e, 0x176d486f, 0x097c13ea, 0x631da5c7, + 0x445f7382, 0x175683f4, 0xcdc66a97, 0x70be0288, 0xb3cdcf72, 0x6e5dd2f3, 0x20936079, 0x459b80a5, + 0xbe60e2db, 0xa9c23101, 0xeba5315c, 0x224e42f2, 0x1c5c1572, 0xf6721b2c, 0x1ad2fff3, 0x8c25404e, + 0x324ed72f, 0x4067b7fd, 0x0523138e, 0x5ca3bc78, 0xdc0fd66e, 0x75922283, 0x784d6b17, 0x58ebb16e, + 0x44094f85, 0x3f481d87, 0xfcfeae7b, 0x77b5ff76, 0x8c2302bf, 0xaaf47556, 0x5f46b02a, 0x2b092801, + 0x3d38f5f7, 0x0ca81f36, 0x52af4a8a, 0x66d5e7c0, 0xdf3b0874, 0x95055110, 0x1b5ad7a8, 0xf61ed5ad, + 0x6cf6e479, 0x20758184, 0xd0cefa65, 0x88f7be58, 0x4a046826, 0x0ff6f8f3, 0xa09c7f70, 0x5346aba0, + 0x5ce96c28, 0xe176eda3, 0x6bac307f, 0x376829d2, 0x85360fa9, 0x17e3fe2a, 0x24b79767, 0xf5a96b20, + 0xd6cd2595, 0x68ff1ebf, 0x7555442c, 0xf19f06be, 0xf9e0659a, 0xeeb9491d, 0x34010718, 0xbb30cab8, + 0xe822fe15, 0x88570983, 0x750e6249, 0xda627e55, 0x5e76ffa8, 0xb1534546, 0x6d47de08, 0xefe9e7d4, + }, + { + 0xf6fa8f9d, 0x2cac6ce1, 0x4ca34867, 0xe2337f7c, 0x95db08e7, 0x016843b4, 0xeced5cbc, 0x325553ac, + 0xbf9f0960, 0xdfa1e2ed, 0x83f0579d, 0x63ed86b9, 0x1ab6a6b8, 0xde5ebe39, 0xf38ff732, 0x8989b138, + 0x33f14961, 0xc01937bd, 0xf506c6da, 0xe4625e7e, 0xa308ea99, 0x4e23e33c, 0x79cbd7cc, 0x48a14367, + 0xa3149619, 0xfec94bd5, 0xa114174a, 0xeaa01866, 0xa084db2d, 0x09a8486f, 0xa888614a, 0x2900af98, + 0x01665991, 0xe1992863, 0xc8f30c60, 0x2e78ef3c, 0xd0d51932, 0xcf0fec14, 0xf7ca07d2, 0xd0a82072, + 0xfd41197e, 0x9305a6b0, 0xe86be3da, 0x74bed3cd, 0x372da53c, 0x4c7f4448, 0xdab5d440, 0x6dba0ec3, + 0x083919a7, 0x9fbaeed9, 0x49dbcfb0, 0x4e670c53, 0x5c3d9c01, 0x64bdb941, 0x2c0e636a, 0xba7dd9cd, + 0xea6f7388, 0xe70bc762, 0x35f29adb, 0x5c4cdd8d, 0xf0d48d8c, 0xb88153e2, 0x08a19866, 0x1ae2eac8, + 0x284caf89, 0xaa928223, 0x9334be53, 0x3b3a21bf, 0x16434be3, 0x9aea3906, 0xefe8c36e, 0xf890cdd9, + 0x80226dae, 0xc340a4a3, 0xdf7e9c09, 0xa694a807, 0x5b7c5ecc, 0x221db3a6, 0x9a69a02f, 0x68818a54, + 0xceb2296f, 0x53c0843a, 0xfe893655, 0x25bfe68a, 0xb4628abc, 0xcf222ebf, 0x25ac6f48, 0xa9a99387, + 0x53bddb65, 0xe76ffbe7, 0xe967fd78, 0x0ba93563, 0x8e342bc1, 0xe8a11be9, 0x4980740d, 0xc8087dfc, + 0x8de4bf99, 0xa11101a0, 0x7fd37975, 0xda5a26c0, 0xe81f994f, 0x9528cd89, 0xfd339fed, 0xb87834bf, + 0x5f04456d, 0x22258698, 0xc9c4c83b, 0x2dc156be, 0x4f628daa, 0x57f55ec5, 0xe2220abe, 0xd2916ebf, + 0x4ec75b95, 0x24f2c3c0, 0x42d15d99, 0xcd0d7fa0, 0x7b6e27ff, 0xa8dc8af0, 0x7345c106, 0xf41e232f, + 0x35162386, 0xe6ea8926, 0x3333b094, 0x157ec6f2, 0x372b74af, 0x692573e4, 0xe9a9d848, 0xf3160289, + 0x3a62ef1d, 0xa787e238, 0xf3a5f676, 0x74364853, 0x20951063, 0x4576698d, 0xb6fad407, 0x592af950, + 0x36f73523, 0x4cfb6e87, 0x7da4cec0, 0x6c152daa, 0xcb0396a8, 0xc50dfe5d, 0xfcd707ab, 0x0921c42f, + 0x89dff0bb, 0x5fe2be78, 0x448f4f33, 0x754613c9, 0x2b05d08d, 0x48b9d585, 0xdc049441, 0xc8098f9b, + 0x7dede786, 0xc39a3373, 0x42410005, 0x6a091751, 0x0ef3c8a6, 0x890072d6, 0x28207682, 0xa9a9f7be, + 0xbf32679d, 0xd45b5b75, 0xb353fd00, 0xcbb0e358, 0x830f220a, 0x1f8fb214, 0xd372cf08, 0xcc3c4a13, + 0x8cf63166, 0x061c87be, 0x88c98f88, 0x6062e397, 0x47cf8e7a, 0xb6c85283, 0x3cc2acfb, 0x3fc06976, + 0x4e8f0252, 0x64d8314d, 0xda3870e3, 0x1e665459, 0xc10908f0, 0x513021a5, 0x6c5b68b7, 0x822f8aa0, + 0x3007cd3e, 0x74719eef, 0xdc872681, 0x073340d4, 0x7e432fd9, 0x0c5ec241, 0x8809286c, 0xf592d891, + 0x08a930f6, 0x957ef305, 0xb7fbffbd, 0xc266e96f, 0x6fe4ac98, 0xb173ecc0, 0xbc60b42a, 0x953498da, + 0xfba1ae12, 0x2d4bd736, 0x0f25faab, 0xa4f3fceb, 0xe2969123, 0x257f0c3d, 0x9348af49, 0x361400bc, + 0xe8816f4a, 0x3814f200, 0xa3f94043, 0x9c7a54c2, 0xbc704f57, 0xda41e7f9, 0xc25ad33a, 0x54f4a084, + 0xb17f5505, 0x59357cbe, 0xedbd15c8, 0x7f97c5ab, 0xba5ac7b5, 0xb6f6deaf, 0x3a479c3a, 0x5302da25, + 0x653d7e6a, 0x54268d49, 0x51a477ea, 0x5017d55b, 0xd7d25d88, 0x44136c76, 0x0404a8c8, 0xb8e5a121, + 0xb81a928a, 0x60ed5869, 0x97c55b96, 0xeaec991b, 0x29935913, 0x01fdb7f1, 0x088e8dfa, 0x9ab6f6f5, + 0x3b4cbf9f, 0x4a5de3ab, 0xe6051d35, 0xa0e1d855, 0xd36b4cf1, 0xf544edeb, 0xb0e93524, 0xbebb8fbd, + 0xa2d762cf, 0x49c92f54, 0x38b5f331, 0x7128a454, 0x48392905, 0xa65b1db8, 0x851c97bd, 0xd675cf2f, + }, + { + 0x85e04019, 0x332bf567, 0x662dbfff, 0xcfc65693, 0x2a8d7f6f, 0xab9bc912, 0xde6008a1, 0x2028da1f, + 0x0227bce7, 0x4d642916, 0x18fac300, 0x50f18b82, 0x2cb2cb11, 0xb232e75c, 0x4b3695f2, 0xb28707de, + 0xa05fbcf6, 0xcd4181e9, 0xe150210c, 0xe24ef1bd, 0xb168c381, 0xfde4e789, 0x5c79b0d8, 0x1e8bfd43, + 0x4d495001, 0x38be4341, 0x913cee1d, 0x92a79c3f, 0x089766be, 0xbaeeadf4, 0x1286becf, 0xb6eacb19, + 0x2660c200, 0x7565bde4, 0x64241f7a, 0x8248dca9, 0xc3b3ad66, 0x28136086, 0x0bd8dfa8, 0x356d1cf2, + 0x107789be, 0xb3b2e9ce, 0x0502aa8f, 0x0bc0351e, 0x166bf52a, 0xeb12ff82, 0xe3486911, 0xd34d7516, + 0x4e7b3aff, 0x5f43671b, 0x9cf6e037, 0x4981ac83, 0x334266ce, 0x8c9341b7, 0xd0d854c0, 0xcb3a6c88, + 0x47bc2829, 0x4725ba37, 0xa66ad22b, 0x7ad61f1e, 0x0c5cbafa, 0x4437f107, 0xb6e79962, 0x42d2d816, + 0x0a961288, 0xe1a5c06e, 0x13749e67, 0x72fc081a, 0xb1d139f7, 0xf9583745, 0xcf19df58, 0xbec3f756, + 0xc06eba30, 0x07211b24, 0x45c28829, 0xc95e317f, 0xbc8ec511, 0x38bc46e9, 0xc6e6fa14, 0xbae8584a, + 0xad4ebc46, 0x468f508b, 0x7829435f, 0xf124183b, 0x821dba9f, 0xaff60ff4, 0xea2c4e6d, 0x16e39264, + 0x92544a8b, 0x009b4fc3, 0xaba68ced, 0x9ac96f78, 0x06a5b79a, 0xb2856e6e, 0x1aec3ca9, 0xbe838688, + 0x0e0804e9, 0x55f1be56, 0xe7e5363b, 0xb3a1f25d, 0xf7debb85, 0x61fe033c, 0x16746233, 0x3c034c28, + 0xda6d0c74, 0x79aac56c, 0x3ce4e1ad, 0x51f0c802, 0x98f8f35a, 0x1626a49f, 0xeed82b29, 0x1d382fe3, + 0x0c4fb99a, 0xbb325778, 0x3ec6d97b, 0x6e77a6a9, 0xcb658b5c, 0xd45230c7, 0x2bd1408b, 0x60c03eb7, + 0xb9068d78, 0xa33754f4, 0xf430c87d, 0xc8a71302, 0xb96d8c32, 0xebd4e7be, 0xbe8b9d2d, 0x7979fb06, + 0xe7225308, 0x8b75cf77, 0x11ef8da4, 0xe083c858, 0x8d6b786f, 0x5a6317a6, 0xfa5cf7a0, 0x5dda0033, + 0xf28ebfb0, 0xf5b9c310, 0xa0eac280, 0x08b9767a, 0xa3d9d2b0, 0x79d34217, 0x021a718d, 0x9ac6336a, + 0x2711fd60, 0x438050e3, 0x069908a8, 0x3d7fedc4, 0x826d2bef, 0x4eeb8476, 0x488dcf25, 0x36c9d566, + 0x28e74e41, 0xc2610aca, 0x3d49a9cf, 0xbae3b9df, 0xb65f8de6, 0x92aeaf64, 0x3ac7d5e6, 0x9ea80509, + 0xf22b017d, 0xa4173f70, 0xdd1e16c3, 0x15e0d7f9, 0x50b1b887, 0x2b9f4fd5, 0x625aba82, 0x6a017962, + 0x2ec01b9c, 0x15488aa9, 0xd716e740, 0x40055a2c, 0x93d29a22, 0xe32dbf9a, 0x058745b9, 0x3453dc1e, + 0xd699296e, 0x496cff6f, 0x1c9f4986, 0xdfe2ed07, 0xb87242d1, 0x19de7eae, 0x053e561a, 0x15ad6f8c, + 0x66626c1c, 0x7154c24c, 0xea082b2a, 0x93eb2939, 0x17dcb0f0, 0x58d4f2ae, 0x9ea294fb, 0x52cf564c, + 0x9883fe66, 0x2ec40581, 0x763953c3, 0x01d6692e, 0xd3a0c108, 0xa1e7160e, 0xe4f2dfa6, 0x693ed285, + 0x74904698, 0x4c2b0edd, 0x4f757656, 0x5d393378, 0xa132234f, 0x3d321c5d, 0xc3f5e194, 0x4b269301, + 0xc79f022f, 0x3c997e7e, 0x5e4f9504, 0x3ffafbbd, 0x76f7ad0e, 0x296693f4, 0x3d1fce6f, 0xc61e45be, + 0xd3b5ab34, 0xf72bf9b7, 0x1b0434c0, 0x4e72b567, 0x5592a33d, 0xb5229301, 0xcfd2a87f, 0x60aeb767, + 0x1814386b, 0x30bcc33d, 0x38a0c07d, 0xfd1606f2, 0xc363519b, 0x589dd390, 0x5479f8e6, 0x1cb8d647, + 0x97fd61a9, 0xea7759f4, 0x2d57539d, 0x569a58cf, 0xe84e63ad, 0x462e1b78, 0x6580f87e, 0xf3817914, + 0x91da55f4, 0x40a230f3, 0xd1988f35, 0xb6e318d2, 0x3ffa50bc, 0x3d40f021, 0xc3c0bdae, 0x4958c24c, + 0x518f36b2, 0x84b1d370, 0x0fedce83, 0x878ddada, 0xf2a279c7, 0x94e01be8, 0x90716f4b, 0x954b8aa3, + }, + { + 0xe216300d, 0xbbddfffc, 0xa7ebdabd, 0x35648095, 0x7789f8b7, 0xe6c1121b, 0x0e241600, 0x052ce8b5, + 0x11a9cfb0, 0xe5952f11, 0xece7990a, 0x9386d174, 0x2a42931c, 0x76e38111, 0xb12def3a, 0x37ddddfc, + 0xde9adeb1, 0x0a0cc32c, 0xbe197029, 0x84a00940, 0xbb243a0f, 0xb4d137cf, 0xb44e79f0, 0x049eedfd, + 0x0b15a15d, 0x480d3168, 0x8bbbde5a, 0x669ded42, 0xc7ece831, 0x3f8f95e7, 0x72df191b, 0x7580330d, + 0x94074251, 0x5c7dcdfa, 0xabbe6d63, 0xaa402164, 0xb301d40a, 0x02e7d1ca, 0x53571dae, 0x7a3182a2, + 0x12a8ddec, 0xfdaa335d, 0x176f43e8, 0x71fb46d4, 0x38129022, 0xce949ad4, 0xb84769ad, 0x965bd862, + 0x82f3d055, 0x66fb9767, 0x15b80b4e, 0x1d5b47a0, 0x4cfde06f, 0xc28ec4b8, 0x57e8726e, 0x647a78fc, + 0x99865d44, 0x608bd593, 0x6c200e03, 0x39dc5ff6, 0x5d0b00a3, 0xae63aff2, 0x7e8bd632, 0x70108c0c, + 0xbbd35049, 0x2998df04, 0x980cf42a, 0x9b6df491, 0x9e7edd53, 0x06918548, 0x58cb7e07, 0x3b74ef2e, + 0x522fffb1, 0xd24708cc, 0x1c7e27cd, 0xa4eb215b, 0x3cf1d2e2, 0x19b47a38, 0x424f7618, 0x35856039, + 0x9d17dee7, 0x27eb35e6, 0xc9aff67b, 0x36baf5b8, 0x09c467cd, 0xc18910b1, 0xe11dbf7b, 0x06cd1af8, + 0x7170c608, 0x2d5e3354, 0xd4de495a, 0x64c6d006, 0xbcc0c62c, 0x3dd00db3, 0x708f8f34, 0x77d51b42, + 0x264f620f, 0x24b8d2bf, 0x15c1b79e, 0x46a52564, 0xf8d7e54e, 0x3e378160, 0x7895cda5, 0x859c15a5, + 0xe6459788, 0xc37bc75f, 0xdb07ba0c, 0x0676a3ab, 0x7f229b1e, 0x31842e7b, 0x24259fd7, 0xf8bef472, + 0x835ffcb8, 0x6df4c1f2, 0x96f5b195, 0xfd0af0fc, 0xb0fe134c, 0xe2506d3d, 0x4f9b12ea, 0xf215f225, + 0xa223736f, 0x9fb4c428, 0x25d04979, 0x34c713f8, 0xc4618187, 0xea7a6e98, 0x7cd16efc, 0x1436876c, + 0xf1544107, 0xbedeee14, 0x56e9af27, 0xa04aa441, 0x3cf7c899, 0x92ecbae6, 0xdd67016d, 0x151682eb, + 0xa842eedf, 0xfdba60b4, 0xf1907b75, 0x20e3030f, 0x24d8c29e, 0xe139673b, 0xefa63fb8, 0x71873054, + 0xb6f2cf3b, 0x9f326442, 0xcb15a4cc, 0xb01a4504, 0xf1e47d8d, 0x844a1be5, 0xbae7dfdc, 0x42cbda70, + 0xcd7dae0a, 0x57e85b7a, 0xd53f5af6, 0x20cf4d8c, 0xcea4d428, 0x79d130a4, 0x3486ebfb, 0x33d3cddc, + 0x77853b53, 0x37effcb5, 0xc5068778, 0xe580b3e6, 0x4e68b8f4, 0xc5c8b37e, 0x0d809ea2, 0x398feb7c, + 0x132a4f94, 0x43b7950e, 0x2fee7d1c, 0x223613bd, 0xdd06caa2, 0x37df932b, 0xc4248289, 0xacf3ebc3, + 0x5715f6b7, 0xef3478dd, 0xf267616f, 0xc148cbe4, 0x9052815e, 0x5e410fab, 0xb48a2465, 0x2eda7fa4, + 0xe87b40e4, 0xe98ea084, 0x5889e9e1, 0xefd390fc, 0xdd07d35b, 0xdb485694, 0x38d7e5b2, 0x57720101, + 0x730edebc, 0x5b643113, 0x94917e4f, 0x503c2fba, 0x646f1282, 0x7523d24a, 0xe0779695, 0xf9c17a8f, + 0x7a5b2121, 0xd187b896, 0x29263a4d, 0xba510cdf, 0x81f47c9f, 0xad1163ed, 0xea7b5965, 0x1a00726e, + 0x11403092, 0x00da6d77, 0x4a0cdd61, 0xad1f4603, 0x605bdfb0, 0x9eedc364, 0x22ebe6a8, 0xcee7d28a, + 0xa0e736a0, 0x5564a6b9, 0x10853209, 0xc7eb8f37, 0x2de705ca, 0x8951570f, 0xdf09822b, 0xbd691a6c, + 0xaa12e4f2, 0x87451c0f, 0xe0f6a27a, 0x3ada4819, 0x4cf1764f, 0x0d771c2b, 0x67cdb156, 0x350d8384, + 0x5938fa0f, 0x42399ef3, 0x36997b07, 0x0e84093d, 0x4aa93e61, 0x8360d87b, 0x1fa98b0c, 0x1149382c, + 0xe97625a5, 0x0614d1b7, 0x0e25244b, 0x0c768347, 0x589e8d82, 0x0d2059d1, 0xa466bb1e, 0xf8da0a82, + 0x04f19130, 0xba6e4ec0, 0x99265164, 0x1ee7230d, 0x50b2ad80, 0xeaee6801, 0x8db2a283, 0xea8bf59e, + }, +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/cast5/cast5_test.go b/Godeps/_workspace/src/golang.org/x/crypto/cast5/cast5_test.go new file mode 100644 index 00000000000..778b272a638 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/cast5/cast5_test.go @@ -0,0 +1,106 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cast5 + +import ( + "bytes" + "encoding/hex" + "testing" +) + +// This test vector is taken from RFC 2144, App B.1. +// Since the other two test vectors are for reduced-round variants, we can't +// use them. +var basicTests = []struct { + key, plainText, cipherText string +}{ + { + "0123456712345678234567893456789a", + "0123456789abcdef", + "238b4fe5847e44b2", + }, +} + +func TestBasic(t *testing.T) { + for i, test := range basicTests { + key, _ := hex.DecodeString(test.key) + plainText, _ := hex.DecodeString(test.plainText) + expected, _ := hex.DecodeString(test.cipherText) + + c, err := NewCipher(key) + if err != nil { + t.Errorf("#%d: failed to create Cipher: %s", i, err) + continue + } + var cipherText [BlockSize]byte + c.Encrypt(cipherText[:], plainText) + if !bytes.Equal(cipherText[:], expected) { + t.Errorf("#%d: got:%x want:%x", i, cipherText, expected) + } + + var plainTextAgain [BlockSize]byte + c.Decrypt(plainTextAgain[:], cipherText[:]) + if !bytes.Equal(plainTextAgain[:], plainText) { + t.Errorf("#%d: got:%x want:%x", i, plainTextAgain, plainText) + } + } +} + +// TestFull performs the test specified in RFC 2144, App B.2. +// However, due to the length of time taken, it's disabled here and a more +// limited version is included, below. +func TestFull(t *testing.T) { + if testing.Short() { + // This is too slow for normal testing + return + } + + a, b := iterate(1000000) + + const expectedA = "eea9d0a249fd3ba6b3436fb89d6dca92" + const expectedB = "b2c95eb00c31ad7180ac05b8e83d696e" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} + +func iterate(iterations int) ([]byte, []byte) { + const initValueHex = "0123456712345678234567893456789a" + + initValue, _ := hex.DecodeString(initValueHex) + + var a, b [16]byte + copy(a[:], initValue) + copy(b[:], initValue) + + for i := 0; i < iterations; i++ { + c, _ := NewCipher(b[:]) + c.Encrypt(a[:8], a[:8]) + c.Encrypt(a[8:], a[8:]) + c, _ = NewCipher(a[:]) + c.Encrypt(b[:8], b[:8]) + c.Encrypt(b[8:], b[8:]) + } + + return a[:], b[:] +} + +func TestLimited(t *testing.T) { + a, b := iterate(1000) + + const expectedA = "23f73b14b02a2ad7dfb9f2c35644798d" + const expectedB = "e5bf37eff14c456a40b21ce369370a9f" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/armor.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/armor.go new file mode 100644 index 00000000000..e8df91af082 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/armor.go @@ -0,0 +1,219 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package armor implements OpenPGP ASCII Armor, see RFC 4880. OpenPGP Armor is +// very similar to PEM except that it has an additional CRC checksum. +package armor + +import ( + "bufio" + "bytes" + "encoding/base64" + "golang.org/x/crypto/openpgp/errors" + "io" +) + +// A Block represents an OpenPGP armored structure. +// +// The encoded form is: +// -----BEGIN Type----- +// Headers +// +// base64-encoded Bytes +// '=' base64 encoded checksum +// -----END Type----- +// where Headers is a possibly empty sequence of Key: Value lines. +// +// Since the armored data can be very large, this package presents a streaming +// interface. +type Block struct { + Type string // The type, taken from the preamble (i.e. "PGP SIGNATURE"). + Header map[string]string // Optional headers. + Body io.Reader // A Reader from which the contents can be read + lReader lineReader + oReader openpgpReader +} + +var ArmorCorrupt error = errors.StructuralError("armor invalid") + +const crc24Init = 0xb704ce +const crc24Poly = 0x1864cfb +const crc24Mask = 0xffffff + +// crc24 calculates the OpenPGP checksum as specified in RFC 4880, section 6.1 +func crc24(crc uint32, d []byte) uint32 { + for _, b := range d { + crc ^= uint32(b) << 16 + for i := 0; i < 8; i++ { + crc <<= 1 + if crc&0x1000000 != 0 { + crc ^= crc24Poly + } + } + } + return crc +} + +var armorStart = []byte("-----BEGIN ") +var armorEnd = []byte("-----END ") +var armorEndOfLine = []byte("-----") + +// lineReader wraps a line based reader. It watches for the end of an armor +// block and records the expected CRC value. +type lineReader struct { + in *bufio.Reader + buf []byte + eof bool + crc uint32 +} + +func (l *lineReader) Read(p []byte) (n int, err error) { + if l.eof { + return 0, io.EOF + } + + if len(l.buf) > 0 { + n = copy(p, l.buf) + l.buf = l.buf[n:] + return + } + + line, isPrefix, err := l.in.ReadLine() + if err != nil { + return + } + if isPrefix { + return 0, ArmorCorrupt + } + + if len(line) == 5 && line[0] == '=' { + // This is the checksum line + var expectedBytes [3]byte + var m int + m, err = base64.StdEncoding.Decode(expectedBytes[0:], line[1:]) + if m != 3 || err != nil { + return + } + l.crc = uint32(expectedBytes[0])<<16 | + uint32(expectedBytes[1])<<8 | + uint32(expectedBytes[2]) + + line, _, err = l.in.ReadLine() + if err != nil && err != io.EOF { + return + } + if !bytes.HasPrefix(line, armorEnd) { + return 0, ArmorCorrupt + } + + l.eof = true + return 0, io.EOF + } + + if len(line) > 96 { + return 0, ArmorCorrupt + } + + n = copy(p, line) + bytesToSave := len(line) - n + if bytesToSave > 0 { + if cap(l.buf) < bytesToSave { + l.buf = make([]byte, 0, bytesToSave) + } + l.buf = l.buf[0:bytesToSave] + copy(l.buf, line[n:]) + } + + return +} + +// openpgpReader passes Read calls to the underlying base64 decoder, but keeps +// a running CRC of the resulting data and checks the CRC against the value +// found by the lineReader at EOF. +type openpgpReader struct { + lReader *lineReader + b64Reader io.Reader + currentCRC uint32 +} + +func (r *openpgpReader) Read(p []byte) (n int, err error) { + n, err = r.b64Reader.Read(p) + r.currentCRC = crc24(r.currentCRC, p[:n]) + + if err == io.EOF { + if r.lReader.crc != uint32(r.currentCRC&crc24Mask) { + return 0, ArmorCorrupt + } + } + + return +} + +// Decode reads a PGP armored block from the given Reader. It will ignore +// leading garbage. If it doesn't find a block, it will return nil, io.EOF. The +// given Reader is not usable after calling this function: an arbitrary amount +// of data may have been read past the end of the block. +func Decode(in io.Reader) (p *Block, err error) { + r := bufio.NewReaderSize(in, 100) + var line []byte + ignoreNext := false + +TryNextBlock: + p = nil + + // Skip leading garbage + for { + ignoreThis := ignoreNext + line, ignoreNext, err = r.ReadLine() + if err != nil { + return + } + if ignoreNext || ignoreThis { + continue + } + line = bytes.TrimSpace(line) + if len(line) > len(armorStart)+len(armorEndOfLine) && bytes.HasPrefix(line, armorStart) { + break + } + } + + p = new(Block) + p.Type = string(line[len(armorStart) : len(line)-len(armorEndOfLine)]) + p.Header = make(map[string]string) + nextIsContinuation := false + var lastKey string + + // Read headers + for { + isContinuation := nextIsContinuation + line, nextIsContinuation, err = r.ReadLine() + if err != nil { + p = nil + return + } + if isContinuation { + p.Header[lastKey] += string(line) + continue + } + line = bytes.TrimSpace(line) + if len(line) == 0 { + break + } + + i := bytes.Index(line, []byte(": ")) + if i == -1 { + goto TryNextBlock + } + lastKey = string(line[:i]) + p.Header[lastKey] = string(line[i+2:]) + } + + p.lReader.in = r + p.oReader.currentCRC = crc24Init + p.oReader.lReader = &p.lReader + p.oReader.b64Reader = base64.NewDecoder(base64.StdEncoding, &p.lReader) + p.Body = &p.oReader + + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/armor_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/armor_test.go new file mode 100644 index 00000000000..9334e94e96c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/armor_test.go @@ -0,0 +1,95 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "bytes" + "hash/adler32" + "io/ioutil" + "testing" +) + +func TestDecodeEncode(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorExample1)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + } + expectedType := "PGP SIGNATURE" + if result.Type != expectedType { + t.Errorf("result.Type: got:%s want:%s", result.Type, expectedType) + } + if len(result.Header) != 1 { + t.Errorf("len(result.Header): got:%d want:1", len(result.Header)) + } + v, ok := result.Header["Version"] + if !ok || v != "GnuPG v1.4.10 (GNU/Linux)" { + t.Errorf("result.Header: got:%#v", result.Header) + } + + contents, err := ioutil.ReadAll(result.Body) + if err != nil { + t.Error(err) + } + + if adler32.Checksum(contents) != 0x27b144be { + t.Errorf("contents: got: %x", contents) + } + + buf = bytes.NewBuffer(nil) + w, err := Encode(buf, result.Type, result.Header) + if err != nil { + t.Error(err) + } + _, err = w.Write(contents) + if err != nil { + t.Error(err) + } + w.Close() + + if !bytes.Equal(buf.Bytes(), []byte(armorExample1)) { + t.Errorf("got: %s\nwant: %s", string(buf.Bytes()), armorExample1) + } +} + +func TestLongHeader(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorLongLine)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + return + } + value, ok := result.Header["Version"] + if !ok { + t.Errorf("missing Version header") + } + if value != longValueExpected { + t.Errorf("got: %s want: %s", value, longValueExpected) + } +} + +const armorExample1 = `-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.10 (GNU/Linux) + +iJwEAAECAAYFAk1Fv/0ACgkQo01+GMIMMbsYTwQAiAw+QAaNfY6WBdplZ/uMAccm +4g+81QPmTSGHnetSb6WBiY13kVzK4HQiZH8JSkmmroMLuGeJwsRTEL4wbjRyUKEt +p1xwUZDECs234F1xiG5enc5SGlRtP7foLBz9lOsjx+LEcA4sTl5/2eZR9zyFZqWW +TxRjs+fJCIFuo71xb1g= +=/teI +-----END PGP SIGNATURE-----` + +const armorLongLine = `-----BEGIN PGP SIGNATURE----- +Version: 0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz + +iQEcBAABAgAGBQJMtFESAAoJEKsQXJGvOPsVj40H/1WW6jaMXv4BW+1ueDSMDwM8 +kx1fLOXbVM5/Kn5LStZNt1jWWnpxdz7eq3uiqeCQjmqUoRde3YbB2EMnnwRbAhpp +cacnAvy9ZQ78OTxUdNW1mhX5bS6q1MTEJnl+DcyigD70HG/yNNQD7sOPMdYQw0TA +byQBwmLwmTsuZsrYqB68QyLHI+DUugn+kX6Hd2WDB62DKa2suoIUIHQQCd/ofwB3 +WfCYInXQKKOSxu2YOg2Eb4kLNhSMc1i9uKUWAH+sdgJh7NBgdoE4MaNtBFkHXRvv +okWuf3+xA9ksp1npSY/mDvgHijmjvtpRDe6iUeqfCn8N9u9CBg8geANgaG8+QA4= +=wfQG +-----END PGP SIGNATURE-----` + +const longValueExpected = "0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz" diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/encode.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/encode.go new file mode 100644 index 00000000000..6f07582c37c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/armor/encode.go @@ -0,0 +1,160 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "encoding/base64" + "io" +) + +var armorHeaderSep = []byte(": ") +var blockEnd = []byte("\n=") +var newline = []byte("\n") +var armorEndOfLineOut = []byte("-----\n") + +// writeSlices writes its arguments to the given Writer. +func writeSlices(out io.Writer, slices ...[]byte) (err error) { + for _, s := range slices { + _, err = out.Write(s) + if err != nil { + return err + } + } + return +} + +// lineBreaker breaks data across several lines, all of the same byte length +// (except possibly the last). Lines are broken with a single '\n'. +type lineBreaker struct { + lineLength int + line []byte + used int + out io.Writer + haveWritten bool +} + +func newLineBreaker(out io.Writer, lineLength int) *lineBreaker { + return &lineBreaker{ + lineLength: lineLength, + line: make([]byte, lineLength), + used: 0, + out: out, + } +} + +func (l *lineBreaker) Write(b []byte) (n int, err error) { + n = len(b) + + if n == 0 { + return + } + + if l.used == 0 && l.haveWritten { + _, err = l.out.Write([]byte{'\n'}) + if err != nil { + return + } + } + + if l.used+len(b) < l.lineLength { + l.used += copy(l.line[l.used:], b) + return + } + + l.haveWritten = true + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + excess := l.lineLength - l.used + l.used = 0 + + _, err = l.out.Write(b[0:excess]) + if err != nil { + return + } + + _, err = l.Write(b[excess:]) + return +} + +func (l *lineBreaker) Close() (err error) { + if l.used > 0 { + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + } + + return +} + +// encoding keeps track of a running CRC24 over the data which has been written +// to it and outputs a OpenPGP checksum when closed, followed by an armor +// trailer. +// +// It's built into a stack of io.Writers: +// encoding -> base64 encoder -> lineBreaker -> out +type encoding struct { + out io.Writer + breaker *lineBreaker + b64 io.WriteCloser + crc uint32 + blockType []byte +} + +func (e *encoding) Write(data []byte) (n int, err error) { + e.crc = crc24(e.crc, data) + return e.b64.Write(data) +} + +func (e *encoding) Close() (err error) { + err = e.b64.Close() + if err != nil { + return + } + e.breaker.Close() + + var checksumBytes [3]byte + checksumBytes[0] = byte(e.crc >> 16) + checksumBytes[1] = byte(e.crc >> 8) + checksumBytes[2] = byte(e.crc) + + var b64ChecksumBytes [4]byte + base64.StdEncoding.Encode(b64ChecksumBytes[:], checksumBytes[:]) + + return writeSlices(e.out, blockEnd, b64ChecksumBytes[:], newline, armorEnd, e.blockType, armorEndOfLine) +} + +// Encode returns a WriteCloser which will encode the data written to it in +// OpenPGP armor. +func Encode(out io.Writer, blockType string, headers map[string]string) (w io.WriteCloser, err error) { + bType := []byte(blockType) + err = writeSlices(out, armorStart, bType, armorEndOfLineOut) + if err != nil { + return + } + + for k, v := range headers { + err = writeSlices(out, []byte(k), armorHeaderSep, []byte(v), newline) + if err != nil { + return + } + } + + _, err = out.Write(newline) + if err != nil { + return + } + + e := &encoding{ + out: out, + breaker: newLineBreaker(out, 64), + crc: crc24Init, + blockType: bType, + } + e.b64 = base64.NewEncoder(base64.StdEncoding, e.breaker) + return e, nil +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/canonical_text.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/canonical_text.go new file mode 100644 index 00000000000..e601e389f12 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/canonical_text.go @@ -0,0 +1,59 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import "hash" + +// NewCanonicalTextHash reformats text written to it into the canonical +// form and then applies the hash h. See RFC 4880, section 5.2.1. +func NewCanonicalTextHash(h hash.Hash) hash.Hash { + return &canonicalTextHash{h, 0} +} + +type canonicalTextHash struct { + h hash.Hash + s int +} + +var newline = []byte{'\r', '\n'} + +func (cth *canonicalTextHash) Write(buf []byte) (int, error) { + start := 0 + + for i, c := range buf { + switch cth.s { + case 0: + if c == '\r' { + cth.s = 1 + } else if c == '\n' { + cth.h.Write(buf[start:i]) + cth.h.Write(newline) + start = i + 1 + } + case 1: + cth.s = 0 + } + } + + cth.h.Write(buf[start:]) + return len(buf), nil +} + +func (cth *canonicalTextHash) Sum(in []byte) []byte { + return cth.h.Sum(in) +} + +func (cth *canonicalTextHash) Reset() { + cth.h.Reset() + cth.s = 0 +} + +func (cth *canonicalTextHash) Size() int { + return cth.h.Size() +} + +func (cth *canonicalTextHash) BlockSize() int { + return cth.h.BlockSize() +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/canonical_text_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/canonical_text_test.go new file mode 100644 index 00000000000..8f3ba2a8814 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/canonical_text_test.go @@ -0,0 +1,52 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "testing" +) + +type recordingHash struct { + buf *bytes.Buffer +} + +func (r recordingHash) Write(b []byte) (n int, err error) { + return r.buf.Write(b) +} + +func (r recordingHash) Sum(in []byte) []byte { + return append(in, r.buf.Bytes()...) +} + +func (r recordingHash) Reset() { + panic("shouldn't be called") +} + +func (r recordingHash) Size() int { + panic("shouldn't be called") +} + +func (r recordingHash) BlockSize() int { + panic("shouldn't be called") +} + +func testCanonicalText(t *testing.T, input, expected string) { + r := recordingHash{bytes.NewBuffer(nil)} + c := NewCanonicalTextHash(r) + c.Write([]byte(input)) + result := c.Sum(nil) + if expected != string(result) { + t.Errorf("input: %x got: %x want: %x", input, result, expected) + } +} + +func TestCanonicalText(t *testing.T) { + testCanonicalText(t, "foo\n", "foo\r\n") + testCanonicalText(t, "foo", "foo") + testCanonicalText(t, "foo\r\n", "foo\r\n") + testCanonicalText(t, "foo\r\nbar", "foo\r\nbar") + testCanonicalText(t, "foo\r\nbar\n\n", "foo\r\nbar\r\n\r\n") +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/clearsign/clearsign.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/clearsign/clearsign.go new file mode 100644 index 00000000000..41a4e4341f0 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/clearsign/clearsign.go @@ -0,0 +1,366 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package clearsign generates and processes OpenPGP, clear-signed data. See +// RFC 4880, section 7. +// +// Clearsigned messages are cryptographically signed, but the contents of the +// message are kept in plaintext so that it can be read without special tools. +package clearsign + +import ( + "bufio" + "bytes" + "crypto" + "hash" + "io" + "net/textproto" + "strconv" + + "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/packet" +) + +// A Block represents a clearsigned message. A signature on a Block can +// be checked by passing Bytes into openpgp.CheckDetachedSignature. +type Block struct { + Headers textproto.MIMEHeader // Optional message headers + Plaintext []byte // The original message text + Bytes []byte // The signed message + ArmoredSignature *armor.Block // The signature block +} + +// start is the marker which denotes the beginning of a clearsigned message. +var start = []byte("\n-----BEGIN PGP SIGNED MESSAGE-----") + +// dashEscape is prefixed to any lines that begin with a hypen so that they +// can't be confused with endText. +var dashEscape = []byte("- ") + +// endText is a marker which denotes the end of the message and the start of +// an armored signature. +var endText = []byte("-----BEGIN PGP SIGNATURE-----") + +// end is a marker which denotes the end of the armored signature. +var end = []byte("\n-----END PGP SIGNATURE-----") + +var crlf = []byte("\r\n") +var lf = byte('\n') + +// getLine returns the first \r\n or \n delineated line from the given byte +// array. The line does not include the \r\n or \n. The remainder of the byte +// array (also not including the new line bytes) is also returned and this will +// always be smaller than the original argument. +func getLine(data []byte) (line, rest []byte) { + i := bytes.Index(data, []byte{'\n'}) + var j int + if i < 0 { + i = len(data) + j = i + } else { + j = i + 1 + if i > 0 && data[i-1] == '\r' { + i-- + } + } + return data[0:i], data[j:] +} + +// Decode finds the first clearsigned message in data and returns it, as well +// as the suffix of data which remains after the message. +func Decode(data []byte) (b *Block, rest []byte) { + // start begins with a newline. However, at the very beginning of + // the byte array, we'll accept the start string without it. + rest = data + if bytes.HasPrefix(data, start[1:]) { + rest = rest[len(start)-1:] + } else if i := bytes.Index(data, start); i >= 0 { + rest = rest[i+len(start):] + } else { + return nil, data + } + + // Consume the start line. + _, rest = getLine(rest) + + var line []byte + b = &Block{ + Headers: make(textproto.MIMEHeader), + } + + // Next come a series of header lines. + for { + // This loop terminates because getLine's second result is + // always smaller than its argument. + if len(rest) == 0 { + return nil, data + } + // An empty line marks the end of the headers. + if line, rest = getLine(rest); len(line) == 0 { + break + } + + i := bytes.Index(line, []byte{':'}) + if i == -1 { + return nil, data + } + + key, val := line[0:i], line[i+1:] + key = bytes.TrimSpace(key) + val = bytes.TrimSpace(val) + b.Headers.Add(string(key), string(val)) + } + + firstLine := true + for { + start := rest + + line, rest = getLine(rest) + if bytes.Equal(line, endText) { + // Back up to the start of the line because armor expects to see the + // header line. + rest = start + break + } + + // The final CRLF isn't included in the hash so we don't write it until + // we've seen the next line. + if firstLine { + firstLine = false + } else { + b.Bytes = append(b.Bytes, crlf...) + } + + if bytes.HasPrefix(line, dashEscape) { + line = line[2:] + } + line = bytes.TrimRight(line, " \t") + b.Bytes = append(b.Bytes, line...) + + b.Plaintext = append(b.Plaintext, line...) + b.Plaintext = append(b.Plaintext, lf) + } + + // We want to find the extent of the armored data (including any newlines at + // the end). + i := bytes.Index(rest, end) + if i == -1 { + return nil, data + } + i += len(end) + for i < len(rest) && (rest[i] == '\r' || rest[i] == '\n') { + i++ + } + armored := rest[:i] + rest = rest[i:] + + var err error + b.ArmoredSignature, err = armor.Decode(bytes.NewBuffer(armored)) + if err != nil { + return nil, data + } + + return b, rest +} + +// A dashEscaper is an io.WriteCloser which processes the body of a clear-signed +// message. The clear-signed message is written to buffered and a hash, suitable +// for signing, is maintained in h. +// +// When closed, an armored signature is created and written to complete the +// message. +type dashEscaper struct { + buffered *bufio.Writer + h hash.Hash + hashType crypto.Hash + + atBeginningOfLine bool + isFirstLine bool + + whitespace []byte + byteBuf []byte // a one byte buffer to save allocations + + privateKey *packet.PrivateKey + config *packet.Config +} + +func (d *dashEscaper) Write(data []byte) (n int, err error) { + for _, b := range data { + d.byteBuf[0] = b + + if d.atBeginningOfLine { + // The final CRLF isn't included in the hash so we have to wait + // until this point (the start of the next line) before writing it. + if !d.isFirstLine { + d.h.Write(crlf) + } + d.isFirstLine = false + + // At the beginning of a line, hyphens have to be escaped. + if b == '-' { + // The signature isn't calculated over the dash-escaped text so + // the escape is only written to buffered. + if _, err = d.buffered.Write(dashEscape); err != nil { + return + } + d.h.Write(d.byteBuf) + d.atBeginningOfLine = false + } else if b == '\n' { + // Nothing to do because we dely writing CRLF to the hash. + } else { + d.h.Write(d.byteBuf) + d.atBeginningOfLine = false + } + if err = d.buffered.WriteByte(b); err != nil { + return + } + } else { + // Any whitespace at the end of the line has to be removed so we + // buffer it until we find out whether there's more on this line. + if b == ' ' || b == '\t' || b == '\r' { + d.whitespace = append(d.whitespace, b) + } else if b == '\n' { + // We got a raw \n. Drop any trailing whitespace and write a + // CRLF. + d.whitespace = d.whitespace[:0] + // We dely writing CRLF to the hash until the start of the + // next line. + if err = d.buffered.WriteByte(b); err != nil { + return + } + d.atBeginningOfLine = true + } else { + // Any buffered whitespace wasn't at the end of the line so + // we need to write it out. + if len(d.whitespace) > 0 { + d.h.Write(d.whitespace) + if _, err = d.buffered.Write(d.whitespace); err != nil { + return + } + d.whitespace = d.whitespace[:0] + } + d.h.Write(d.byteBuf) + if err = d.buffered.WriteByte(b); err != nil { + return + } + } + } + } + + n = len(data) + return +} + +func (d *dashEscaper) Close() (err error) { + if !d.atBeginningOfLine { + if err = d.buffered.WriteByte(lf); err != nil { + return + } + } + sig := new(packet.Signature) + sig.SigType = packet.SigTypeText + sig.PubKeyAlgo = d.privateKey.PubKeyAlgo + sig.Hash = d.hashType + sig.CreationTime = d.config.Now() + sig.IssuerKeyId = &d.privateKey.KeyId + + if err = sig.Sign(d.h, d.privateKey, d.config); err != nil { + return + } + + out, err := armor.Encode(d.buffered, "PGP SIGNATURE", nil) + if err != nil { + return + } + + if err = sig.Serialize(out); err != nil { + return + } + if err = out.Close(); err != nil { + return + } + if err = d.buffered.Flush(); err != nil { + return + } + return +} + +// Encode returns a WriteCloser which will clear-sign a message with privateKey +// and write it to w. If config is nil, sensible defaults are used. +func Encode(w io.Writer, privateKey *packet.PrivateKey, config *packet.Config) (plaintext io.WriteCloser, err error) { + if privateKey.Encrypted { + return nil, errors.InvalidArgumentError("signing key is encrypted") + } + + hashType := config.Hash() + name := nameOfHash(hashType) + if len(name) == 0 { + return nil, errors.UnsupportedError("unknown hash type: " + strconv.Itoa(int(hashType))) + } + + if !hashType.Available() { + return nil, errors.UnsupportedError("unsupported hash type: " + strconv.Itoa(int(hashType))) + } + h := hashType.New() + + buffered := bufio.NewWriter(w) + // start has a \n at the beginning that we don't want here. + if _, err = buffered.Write(start[1:]); err != nil { + return + } + if err = buffered.WriteByte(lf); err != nil { + return + } + if _, err = buffered.WriteString("Hash: "); err != nil { + return + } + if _, err = buffered.WriteString(name); err != nil { + return + } + if err = buffered.WriteByte(lf); err != nil { + return + } + if err = buffered.WriteByte(lf); err != nil { + return + } + + plaintext = &dashEscaper{ + buffered: buffered, + h: h, + hashType: hashType, + + atBeginningOfLine: true, + isFirstLine: true, + + byteBuf: make([]byte, 1), + + privateKey: privateKey, + config: config, + } + + return +} + +// nameOfHash returns the OpenPGP name for the given hash, or the empty string +// if the name isn't known. See RFC 4880, section 9.4. +func nameOfHash(h crypto.Hash) string { + switch h { + case crypto.MD5: + return "MD5" + case crypto.SHA1: + return "SHA1" + case crypto.RIPEMD160: + return "RIPEMD160" + case crypto.SHA224: + return "SHA224" + case crypto.SHA256: + return "SHA256" + case crypto.SHA384: + return "SHA384" + case crypto.SHA512: + return "SHA512" + } + return "" +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/clearsign/clearsign_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/clearsign/clearsign_test.go new file mode 100644 index 00000000000..51538b9765e --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/clearsign/clearsign_test.go @@ -0,0 +1,187 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package clearsign + +import ( + "bytes" + "golang.org/x/crypto/openpgp" + "testing" +) + +func testParse(t *testing.T, input []byte, expected, expectedPlaintext string) { + b, rest := Decode(input) + if b == nil { + t.Fatal("failed to decode clearsign message") + } + if !bytes.Equal(rest, []byte("trailing")) { + t.Errorf("unexpected remaining bytes returned: %s", string(rest)) + } + if b.ArmoredSignature.Type != "PGP SIGNATURE" { + t.Errorf("bad armor type, got:%s, want:PGP SIGNATURE", b.ArmoredSignature.Type) + } + if !bytes.Equal(b.Bytes, []byte(expected)) { + t.Errorf("bad body, got:%x want:%x", b.Bytes, expected) + } + + if !bytes.Equal(b.Plaintext, []byte(expectedPlaintext)) { + t.Errorf("bad plaintext, got:%x want:%x", b.Plaintext, expectedPlaintext) + } + + keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(signingKey)) + if err != nil { + t.Errorf("failed to parse public key: %s", err) + } + + if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewBuffer(b.Bytes), b.ArmoredSignature.Body); err != nil { + t.Errorf("failed to check signature: %s", err) + } +} + +func TestParse(t *testing.T) { + testParse(t, clearsignInput, "Hello world\r\nline 2", "Hello world\nline 2\n") + testParse(t, clearsignInput2, "\r\n\r\n(This message has a couple of blank lines at the start and end.)\r\n\r\n", "\n\n(This message has a couple of blank lines at the start and end.)\n\n\n") +} + +func TestParseWithNoNewlineAtEnd(t *testing.T) { + input := clearsignInput + input = input[:len(input)-len("trailing")-1] + b, rest := Decode(input) + if b == nil { + t.Fatal("failed to decode clearsign message") + } + if len(rest) > 0 { + t.Errorf("unexpected remaining bytes returned: %s", string(rest)) + } +} + +var signingTests = []struct { + in, signed, plaintext string +}{ + {"", "", ""}, + {"a", "a", "a\n"}, + {"a\n", "a", "a\n"}, + {"-a\n", "-a", "-a\n"}, + {"--a\nb", "--a\r\nb", "--a\nb\n"}, +} + +func TestSigning(t *testing.T) { + keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(signingKey)) + if err != nil { + t.Errorf("failed to parse public key: %s", err) + } + + for i, test := range signingTests { + var buf bytes.Buffer + + plaintext, err := Encode(&buf, keyring[0].PrivateKey, nil) + if err != nil { + t.Errorf("#%d: error from Encode: %s", i, err) + continue + } + if _, err := plaintext.Write([]byte(test.in)); err != nil { + t.Errorf("#%d: error from Write: %s", i, err) + continue + } + if err := plaintext.Close(); err != nil { + t.Fatalf("#%d: error from Close: %s", i, err) + continue + } + + b, _ := Decode(buf.Bytes()) + if b == nil { + t.Errorf("#%d: failed to decode clearsign message", i) + continue + } + if !bytes.Equal(b.Bytes, []byte(test.signed)) { + t.Errorf("#%d: bad result, got:%x, want:%x", i, b.Bytes, test.signed) + continue + } + if !bytes.Equal(b.Plaintext, []byte(test.plaintext)) { + t.Errorf("#%d: bad result, got:%x, want:%x", i, b.Plaintext, test.plaintext) + continue + } + + if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewBuffer(b.Bytes), b.ArmoredSignature.Body); err != nil { + t.Errorf("#%d: failed to check signature: %s", i, err) + } + } +} + +var clearsignInput = []byte(` +;lasjlkfdsa + +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +Hello world +line 2 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.10 (GNU/Linux) + +iJwEAQECAAYFAk8kMuEACgkQO9o98PRieSpMsAQAhmY/vwmNpflrPgmfWsYhk5O8 +pjnBUzZwqTDoDeINjZEoPDSpQAHGhjFjgaDx/Gj4fAl0dM4D0wuUEBb6QOrwflog +2A2k9kfSOMOtk0IH/H5VuFN1Mie9L/erYXjTQIptv9t9J7NoRBMU0QOOaFU0JaO9 +MyTpno24AjIAGb+mH1U= +=hIJ6 +-----END PGP SIGNATURE----- +trailing`) + +var clearsignInput2 = []byte(` +asdlfkjasdlkfjsadf + +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA256 + + + +(This message has a couple of blank lines at the start and end.) + + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.11 (GNU/Linux) + +iJwEAQEIAAYFAlPpSREACgkQO9o98PRieSpZTAP+M8QUoCt/7Rf3YbXPcdzIL32v +pt1I+cMNeopzfLy0u4ioEFi8s5VkwpL1AFmirvgViCwlf82inoRxzZRiW05JQ5LI +ESEzeCoy2LIdRCQ2hcrG8pIUPzUO4TqO5D/dMbdHwNH4h5nNmGJUAEG6FpURlPm+ +qZg6BaTvOxepqOxnhVU= +=e+C6 +-----END PGP SIGNATURE----- + +trailing`) + +var signingKey = `-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBE2rFNoBBADFwqWQIW/DSqcB4yCQqnAFTJ27qS5AnB46ccAdw3u4Greeu3Bp +idpoHdjULy7zSKlwR1EA873dO/k/e11Ml3dlAFUinWeejWaK2ugFP6JjiieSsrKn +vWNicdCS4HTWn0X4sjl0ZiAygw6GNhqEQ3cpLeL0g8E9hnYzJKQ0LWJa0QARAQAB +AAP/TB81EIo2VYNmTq0pK1ZXwUpxCrvAAIG3hwKjEzHcbQznsjNvPUihZ+NZQ6+X +0HCfPAdPkGDCLCb6NavcSW+iNnLTrdDnSI6+3BbIONqWWdRDYJhqZCkqmG6zqSfL +IdkJgCw94taUg5BWP/AAeQrhzjChvpMQTVKQL5mnuZbUCeMCAN5qrYMP2S9iKdnk +VANIFj7656ARKt/nf4CBzxcpHTyB8+d2CtPDKCmlJP6vL8t58Jmih+kHJMvC0dzn +gr5f5+sCAOOe5gt9e0am7AvQWhdbHVfJU0TQJx+m2OiCJAqGTB1nvtBLHdJnfdC9 +TnXXQ6ZXibqLyBies/xeY2sCKL5qtTMCAKnX9+9d/5yQxRyrQUHt1NYhaXZnJbHx +q4ytu0eWz+5i68IYUSK69jJ1NWPM0T6SkqpB3KCAIv68VFm9PxqG1KmhSrQIVGVz +dCBLZXmIuAQTAQIAIgUCTasU2gIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AA +CgkQO9o98PRieSoLhgQAkLEZex02Qt7vGhZzMwuN0R22w3VwyYyjBx+fM3JFETy1 +ut4xcLJoJfIaF5ZS38UplgakHG0FQ+b49i8dMij0aZmDqGxrew1m4kBfjXw9B/v+ +eIqpODryb6cOSwyQFH0lQkXC040pjq9YqDsO5w0WYNXYKDnzRV0p4H1pweo2VDid +AdgETasU2gEEAN46UPeWRqKHvA99arOxee38fBt2CI08iiWyI8T3J6ivtFGixSqV +bRcPxYO/qLpVe5l84Nb3X71GfVXlc9hyv7CD6tcowL59hg1E/DC5ydI8K8iEpUmK +/UnHdIY5h8/kqgGxkY/T/hgp5fRQgW1ZoZxLajVlMRZ8W4tFtT0DeA+JABEBAAEA +A/0bE1jaaZKj6ndqcw86jd+QtD1SF+Cf21CWRNeLKnUds4FRRvclzTyUMuWPkUeX +TaNNsUOFqBsf6QQ2oHUBBK4VCHffHCW4ZEX2cd6umz7mpHW6XzN4DECEzOVksXtc +lUC1j4UB91DC/RNQqwX1IV2QLSwssVotPMPqhOi0ZLNY7wIA3n7DWKInxYZZ4K+6 +rQ+POsz6brEoRHwr8x6XlHenq1Oki855pSa1yXIARoTrSJkBtn5oI+f8AzrnN0BN +oyeQAwIA/7E++3HDi5aweWrViiul9cd3rcsS0dEnksPhvS0ozCJiHsq/6GFmy7J8 +QSHZPteedBnZyNp5jR+H7cIfVN3KgwH/Skq4PsuPhDq5TKK6i8Pc1WW8MA6DXTdU +nLkX7RGmMwjC0DBf7KWAlPjFaONAX3a8ndnz//fy1q7u2l9AZwrj1qa1iJ8EGAEC +AAkFAk2rFNoCGwwACgkQO9o98PRieSo2/QP/WTzr4ioINVsvN1akKuekmEMI3LAp +BfHwatufxxP1U+3Si/6YIk7kuPB9Hs+pRqCXzbvPRrI8NHZBmc8qIGthishdCYad +AHcVnXjtxrULkQFGbGvhKURLvS9WnzD/m1K2zzwxzkPTzT9/Yf06O6Mal5AdugPL +VrM0m72/jnpKo04= +=zNCn +-----END PGP PRIVATE KEY BLOCK----- +` diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/elgamal/elgamal.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/elgamal/elgamal.go new file mode 100644 index 00000000000..a553bdee8dd --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/elgamal/elgamal.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elgamal implements ElGamal encryption, suitable for OpenPGP, +// as specified in "A Public-Key Cryptosystem and a Signature Scheme Based on +// Discrete Logarithms," IEEE Transactions on Information Theory, v. IT-31, +// n. 4, 1985, pp. 469-472. +// +// This form of ElGamal embeds PKCS#1 v1.5 padding, which may make it +// unsuitable for other protocols. RSA should be used in preference in any +// case. +package elgamal + +import ( + "crypto/rand" + "crypto/subtle" + "errors" + "io" + "math/big" +) + +// PublicKey represents an ElGamal public key. +type PublicKey struct { + G, P, Y *big.Int +} + +// PrivateKey represents an ElGamal private key. +type PrivateKey struct { + PublicKey + X *big.Int +} + +// Encrypt encrypts the given message to the given public key. The result is a +// pair of integers. Errors can result from reading random, or because msg is +// too large to be encrypted to the public key. +func Encrypt(random io.Reader, pub *PublicKey, msg []byte) (c1, c2 *big.Int, err error) { + pLen := (pub.P.BitLen() + 7) / 8 + if len(msg) > pLen-11 { + err = errors.New("elgamal: message too long") + return + } + + // EM = 0x02 || PS || 0x00 || M + em := make([]byte, pLen-1) + em[0] = 2 + ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):] + err = nonZeroRandomBytes(ps, random) + if err != nil { + return + } + em[len(em)-len(msg)-1] = 0 + copy(mm, msg) + + m := new(big.Int).SetBytes(em) + + k, err := rand.Int(random, pub.P) + if err != nil { + return + } + + c1 = new(big.Int).Exp(pub.G, k, pub.P) + s := new(big.Int).Exp(pub.Y, k, pub.P) + c2 = s.Mul(s, m) + c2.Mod(c2, pub.P) + + return +} + +// Decrypt takes two integers, resulting from an ElGamal encryption, and +// returns the plaintext of the message. An error can result only if the +// ciphertext is invalid. Users should keep in mind that this is a padding +// oracle and thus, if exposed to an adaptive chosen ciphertext attack, can +// be used to break the cryptosystem. See ``Chosen Ciphertext Attacks +// Against Protocols Based on the RSA Encryption Standard PKCS #1'', Daniel +// Bleichenbacher, Advances in Cryptology (Crypto '98), +func Decrypt(priv *PrivateKey, c1, c2 *big.Int) (msg []byte, err error) { + s := new(big.Int).Exp(c1, priv.X, priv.P) + s.ModInverse(s, priv.P) + s.Mul(s, c2) + s.Mod(s, priv.P) + em := s.Bytes() + + firstByteIsTwo := subtle.ConstantTimeByteEq(em[0], 2) + + // The remainder of the plaintext must be a string of non-zero random + // octets, followed by a 0, followed by the message. + // lookingForIndex: 1 iff we are still looking for the zero. + // index: the offset of the first zero byte. + var lookingForIndex, index int + lookingForIndex = 1 + + for i := 1; i < len(em); i++ { + equals0 := subtle.ConstantTimeByteEq(em[i], 0) + index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex) + } + + if firstByteIsTwo != 1 || lookingForIndex != 0 || index < 9 { + return nil, errors.New("elgamal: decryption error") + } + return em[index+1:], nil +} + +// nonZeroRandomBytes fills the given slice with non-zero random octets. +func nonZeroRandomBytes(s []byte, rand io.Reader) (err error) { + _, err = io.ReadFull(rand, s) + if err != nil { + return + } + + for i := 0; i < len(s); i++ { + for s[i] == 0 { + _, err = io.ReadFull(rand, s[i:i+1]) + if err != nil { + return + } + } + } + + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/elgamal/elgamal_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/elgamal/elgamal_test.go new file mode 100644 index 00000000000..c4f99f5c48c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/elgamal/elgamal_test.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elgamal + +import ( + "bytes" + "crypto/rand" + "math/big" + "testing" +) + +// This is the 1024-bit MODP group from RFC 5114, section 2.1: +const primeHex = "B10B8F96A080E01DDE92DE5EAE5D54EC52C99FBCFB06A3C69A6A9DCA52D23B616073E28675A23D189838EF1E2EE652C013ECB4AEA906112324975C3CD49B83BFACCBDD7D90C4BD7098488E9C219A73724EFFD6FAE5644738FAA31A4FF55BCCC0A151AF5F0DC8B4BD45BF37DF365C1A65E68CFDA76D4DA708DF1FB2BC2E4A4371" + +const generatorHex = "A4D1CBD5C3FD34126765A442EFB99905F8104DD258AC507FD6406CFF14266D31266FEA1E5C41564B777E690F5504F213160217B4B01B886A5E91547F9E2749F4D7FBD7D3B9A92EE1909D0D2263F80A76A6A24C087A091F531DBF0A0169B6A28AD662A4D18E73AFA32D779D5918D08BC8858F4DCEF97C2A24855E6EEB22B3B2E5" + +func fromHex(hex string) *big.Int { + n, ok := new(big.Int).SetString(hex, 16) + if !ok { + panic("failed to parse hex number") + } + return n +} + +func TestEncryptDecrypt(t *testing.T) { + priv := &PrivateKey{ + PublicKey: PublicKey{ + G: fromHex(generatorHex), + P: fromHex(primeHex), + }, + X: fromHex("42"), + } + priv.Y = new(big.Int).Exp(priv.G, priv.X, priv.P) + + message := []byte("hello world") + c1, c2, err := Encrypt(rand.Reader, &priv.PublicKey, message) + if err != nil { + t.Errorf("error encrypting: %s", err) + } + message2, err := Decrypt(priv, c1, c2) + if err != nil { + t.Errorf("error decrypting: %s", err) + } + if !bytes.Equal(message2, message) { + t.Errorf("decryption failed, got: %x, want: %x", message2, message) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/errors/errors.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/errors/errors.go new file mode 100644 index 00000000000..6c4c2f1769c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/errors/errors.go @@ -0,0 +1,72 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package errors contains common error types for the OpenPGP packages. +package errors + +import ( + "strconv" +) + +// A StructuralError is returned when OpenPGP data is found to be syntactically +// invalid. +type StructuralError string + +func (s StructuralError) Error() string { + return "openpgp: invalid data: " + string(s) +} + +// UnsupportedError indicates that, although the OpenPGP data is valid, it +// makes use of currently unimplemented features. +type UnsupportedError string + +func (s UnsupportedError) Error() string { + return "openpgp: unsupported feature: " + string(s) +} + +// InvalidArgumentError indicates that the caller is in error and passed an +// incorrect value. +type InvalidArgumentError string + +func (i InvalidArgumentError) Error() string { + return "openpgp: invalid argument: " + string(i) +} + +// SignatureError indicates that a syntactically valid signature failed to +// validate. +type SignatureError string + +func (b SignatureError) Error() string { + return "openpgp: invalid signature: " + string(b) +} + +type keyIncorrectError int + +func (ki keyIncorrectError) Error() string { + return "openpgp: incorrect key" +} + +var ErrKeyIncorrect error = keyIncorrectError(0) + +type unknownIssuerError int + +func (unknownIssuerError) Error() string { + return "openpgp: signature made by unknown entity" +} + +var ErrUnknownIssuer error = unknownIssuerError(0) + +type keyRevokedError int + +func (keyRevokedError) Error() string { + return "openpgp: signature made by revoked key" +} + +var ErrKeyRevoked error = keyRevokedError(0) + +type UnknownPacketTypeError uint8 + +func (upte UnknownPacketTypeError) Error() string { + return "openpgp: unknown packet type: " + strconv.Itoa(int(upte)) +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/keys.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/keys.go new file mode 100644 index 00000000000..c4a6640fb82 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/keys.go @@ -0,0 +1,624 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto/rsa" + "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/packet" + "io" + "time" +) + +// PublicKeyType is the armor type for a PGP public key. +var PublicKeyType = "PGP PUBLIC KEY BLOCK" + +// PrivateKeyType is the armor type for a PGP private key. +var PrivateKeyType = "PGP PRIVATE KEY BLOCK" + +// An Entity represents the components of an OpenPGP key: a primary public key +// (which must be a signing key), one or more identities claimed by that key, +// and zero or more subkeys, which may be encryption keys. +type Entity struct { + PrimaryKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Identities map[string]*Identity // indexed by Identity.Name + Revocations []*packet.Signature + Subkeys []Subkey +} + +// An Identity represents an identity claimed by an Entity and zero or more +// assertions by other entities about that claim. +type Identity struct { + Name string // by convention, has the form "Full Name (comment) " + UserId *packet.UserId + SelfSignature *packet.Signature + Signatures []*packet.Signature +} + +// A Subkey is an additional public key in an Entity. Subkeys can be used for +// encryption. +type Subkey struct { + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Sig *packet.Signature +} + +// A Key identifies a specific public key in an Entity. This is either the +// Entity's primary key or a subkey. +type Key struct { + Entity *Entity + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + SelfSignature *packet.Signature +} + +// A KeyRing provides access to public and private keys. +type KeyRing interface { + // KeysById returns the set of keys that have the given key id. + KeysById(id uint64) []Key + // KeysByIdAndUsage returns the set of keys with the given id + // that also meet the key usage given by requiredUsage. + // The requiredUsage is expressed as the bitwise-OR of + // packet.KeyFlag* values. + KeysByIdUsage(id uint64, requiredUsage byte) []Key + // DecryptionKeys returns all private keys that are valid for + // decryption. + DecryptionKeys() []Key +} + +// primaryIdentity returns the Identity marked as primary or the first identity +// if none are so marked. +func (e *Entity) primaryIdentity() *Identity { + var firstIdentity *Identity + for _, ident := range e.Identities { + if firstIdentity == nil { + firstIdentity = ident + } + if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + return ident + } + } + return firstIdentity +} + +// encryptionKey returns the best candidate Key for encrypting a message to the +// given Entity. +func (e *Entity) encryptionKey(now time.Time) (Key, bool) { + candidateSubkey := -1 + + for i, subkey := range e.Subkeys { + if subkey.Sig.FlagsValid && + subkey.Sig.FlagEncryptCommunications && + subkey.PublicKey.PubKeyAlgo.CanEncrypt() && + !subkey.Sig.KeyExpired(now) { + candidateSubkey = i + break + } + } + + if candidateSubkey != -1 { + subkey := e.Subkeys[candidateSubkey] + return Key{e, subkey.PublicKey, subkey.PrivateKey, subkey.Sig}, true + } + + // If we don't have any candidate subkeys for encryption and + // the primary key doesn't have any usage metadata then we + // assume that the primary key is ok. Or, if the primary key is + // marked as ok to encrypt to, then we can obviously use it. + i := e.primaryIdentity() + if !i.SelfSignature.FlagsValid || i.SelfSignature.FlagEncryptCommunications && + e.PrimaryKey.PubKeyAlgo.CanEncrypt() && + !i.SelfSignature.KeyExpired(now) { + return Key{e, e.PrimaryKey, e.PrivateKey, i.SelfSignature}, true + } + + // This Entity appears to be signing only. + return Key{}, false +} + +// signingKey return the best candidate Key for signing a message with this +// Entity. +func (e *Entity) signingKey(now time.Time) (Key, bool) { + candidateSubkey := -1 + + for i, subkey := range e.Subkeys { + if subkey.Sig.FlagsValid && + subkey.Sig.FlagSign && + subkey.PublicKey.PubKeyAlgo.CanSign() && + !subkey.Sig.KeyExpired(now) { + candidateSubkey = i + break + } + } + + if candidateSubkey != -1 { + subkey := e.Subkeys[candidateSubkey] + return Key{e, subkey.PublicKey, subkey.PrivateKey, subkey.Sig}, true + } + + // If we have no candidate subkey then we assume that it's ok to sign + // with the primary key. + i := e.primaryIdentity() + if !i.SelfSignature.FlagsValid || i.SelfSignature.FlagSign && + !i.SelfSignature.KeyExpired(now) { + return Key{e, e.PrimaryKey, e.PrivateKey, i.SelfSignature}, true + } + + return Key{}, false +} + +// An EntityList contains one or more Entities. +type EntityList []*Entity + +// KeysById returns the set of keys that have the given key id. +func (el EntityList) KeysById(id uint64) (keys []Key) { + for _, e := range el { + if e.PrimaryKey.KeyId == id { + var selfSig *packet.Signature + for _, ident := range e.Identities { + if selfSig == nil { + selfSig = ident.SelfSignature + } else if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + selfSig = ident.SelfSignature + break + } + } + keys = append(keys, Key{e, e.PrimaryKey, e.PrivateKey, selfSig}) + } + + for _, subKey := range e.Subkeys { + if subKey.PublicKey.KeyId == id { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// KeysByIdAndUsage returns the set of keys with the given id that also meet +// the key usage given by requiredUsage. The requiredUsage is expressed as +// the bitwise-OR of packet.KeyFlag* values. +func (el EntityList) KeysByIdUsage(id uint64, requiredUsage byte) (keys []Key) { + for _, key := range el.KeysById(id) { + if len(key.Entity.Revocations) > 0 { + continue + } + + if key.SelfSignature.RevocationReason != nil { + continue + } + + if key.SelfSignature.FlagsValid && requiredUsage != 0 { + var usage byte + if key.SelfSignature.FlagCertify { + usage |= packet.KeyFlagCertify + } + if key.SelfSignature.FlagSign { + usage |= packet.KeyFlagSign + } + if key.SelfSignature.FlagEncryptCommunications { + usage |= packet.KeyFlagEncryptCommunications + } + if key.SelfSignature.FlagEncryptStorage { + usage |= packet.KeyFlagEncryptStorage + } + if usage&requiredUsage != requiredUsage { + continue + } + } + + keys = append(keys, key) + } + return +} + +// DecryptionKeys returns all private keys that are valid for decryption. +func (el EntityList) DecryptionKeys() (keys []Key) { + for _, e := range el { + for _, subKey := range e.Subkeys { + if subKey.PrivateKey != nil && (!subKey.Sig.FlagsValid || subKey.Sig.FlagEncryptStorage || subKey.Sig.FlagEncryptCommunications) { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// ReadArmoredKeyRing reads one or more public/private keys from an armor keyring file. +func ReadArmoredKeyRing(r io.Reader) (EntityList, error) { + block, err := armor.Decode(r) + if err == io.EOF { + return nil, errors.InvalidArgumentError("no armored data found") + } + if err != nil { + return nil, err + } + if block.Type != PublicKeyType && block.Type != PrivateKeyType { + return nil, errors.InvalidArgumentError("expected public or private key block, got: " + block.Type) + } + + return ReadKeyRing(block.Body) +} + +// ReadKeyRing reads one or more public/private keys. Unsupported keys are +// ignored as long as at least a single valid key is found. +func ReadKeyRing(r io.Reader) (el EntityList, err error) { + packets := packet.NewReader(r) + var lastUnsupportedError error + + for { + var e *Entity + e, err = ReadEntity(packets) + if err != nil { + // TODO: warn about skipped unsupported/unreadable keys + if _, ok := err.(errors.UnsupportedError); ok { + lastUnsupportedError = err + err = readToNextPublicKey(packets) + } else if _, ok := err.(errors.StructuralError); ok { + // Skip unreadable, badly-formatted keys + lastUnsupportedError = err + err = readToNextPublicKey(packets) + } + if err == io.EOF { + err = nil + break + } + if err != nil { + el = nil + break + } + } else { + el = append(el, e) + } + } + + if len(el) == 0 && err == nil { + err = lastUnsupportedError + } + return +} + +// readToNextPublicKey reads packets until the start of the entity and leaves +// the first packet of the new entity in the Reader. +func readToNextPublicKey(packets *packet.Reader) (err error) { + var p packet.Packet + for { + p, err = packets.Next() + if err == io.EOF { + return + } else if err != nil { + if _, ok := err.(errors.UnsupportedError); ok { + err = nil + continue + } + return + } + + if pk, ok := p.(*packet.PublicKey); ok && !pk.IsSubkey { + packets.Unread(p) + return + } + } + + panic("unreachable") +} + +// ReadEntity reads an entity (public key, identities, subkeys etc) from the +// given Reader. +func ReadEntity(packets *packet.Reader) (*Entity, error) { + e := new(Entity) + e.Identities = make(map[string]*Identity) + + p, err := packets.Next() + if err != nil { + return nil, err + } + + var ok bool + if e.PrimaryKey, ok = p.(*packet.PublicKey); !ok { + if e.PrivateKey, ok = p.(*packet.PrivateKey); !ok { + packets.Unread(p) + return nil, errors.StructuralError("first packet was not a public/private key") + } else { + e.PrimaryKey = &e.PrivateKey.PublicKey + } + } + + if !e.PrimaryKey.PubKeyAlgo.CanSign() { + return nil, errors.StructuralError("primary key cannot be used for signatures") + } + + var current *Identity + var revocations []*packet.Signature +EachPacket: + for { + p, err := packets.Next() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + switch pkt := p.(type) { + case *packet.UserId: + current = new(Identity) + current.Name = pkt.Id + current.UserId = pkt + e.Identities[pkt.Id] = current + + for { + p, err = packets.Next() + if err == io.EOF { + return nil, io.ErrUnexpectedEOF + } else if err != nil { + return nil, err + } + + sig, ok := p.(*packet.Signature) + if !ok { + return nil, errors.StructuralError("user ID packet not followed by self-signature") + } + + if (sig.SigType == packet.SigTypePositiveCert || sig.SigType == packet.SigTypeGenericCert) && sig.IssuerKeyId != nil && *sig.IssuerKeyId == e.PrimaryKey.KeyId { + if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, e.PrimaryKey, sig); err != nil { + return nil, errors.StructuralError("user ID self-signature invalid: " + err.Error()) + } + current.SelfSignature = sig + break + } + current.Signatures = append(current.Signatures, sig) + } + case *packet.Signature: + if pkt.SigType == packet.SigTypeKeyRevocation { + revocations = append(revocations, pkt) + } else if pkt.SigType == packet.SigTypeDirectSignature { + // TODO: RFC4880 5.2.1 permits signatures + // directly on keys (eg. to bind additional + // revocation keys). + } else if current == nil { + return nil, errors.StructuralError("signature packet found before user id packet") + } else { + current.Signatures = append(current.Signatures, pkt) + } + case *packet.PrivateKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, &pkt.PublicKey, pkt) + if err != nil { + return nil, err + } + case *packet.PublicKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, pkt, nil) + if err != nil { + return nil, err + } + default: + // we ignore unknown packets + } + } + + if len(e.Identities) == 0 { + return nil, errors.StructuralError("entity without any identities") + } + + for _, revocation := range revocations { + err = e.PrimaryKey.VerifyRevocationSignature(revocation) + if err == nil { + e.Revocations = append(e.Revocations, revocation) + } else { + // TODO: RFC 4880 5.2.3.15 defines revocation keys. + return nil, errors.StructuralError("revocation signature signed by alternate key") + } + } + + return e, nil +} + +func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *packet.PrivateKey) error { + var subKey Subkey + subKey.PublicKey = pub + subKey.PrivateKey = priv + p, err := packets.Next() + if err == io.EOF { + return io.ErrUnexpectedEOF + } + if err != nil { + return errors.StructuralError("subkey signature invalid: " + err.Error()) + } + var ok bool + subKey.Sig, ok = p.(*packet.Signature) + if !ok { + return errors.StructuralError("subkey packet not followed by signature") + } + if subKey.Sig.SigType != packet.SigTypeSubkeyBinding && subKey.Sig.SigType != packet.SigTypeSubkeyRevocation { + return errors.StructuralError("subkey signature with wrong type") + } + err = e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, subKey.Sig) + if err != nil { + return errors.StructuralError("subkey signature invalid: " + err.Error()) + } + e.Subkeys = append(e.Subkeys, subKey) + return nil +} + +const defaultRSAKeyBits = 2048 + +// NewEntity returns an Entity that contains a fresh RSA/RSA keypair with a +// single identity composed of the given full name, comment and email, any of +// which may be empty but must not contain any of "()<>\x00". +// If config is nil, sensible defaults will be used. +func NewEntity(name, comment, email string, config *packet.Config) (*Entity, error) { + currentTime := config.Now() + + uid := packet.NewUserId(name, comment, email) + if uid == nil { + return nil, errors.InvalidArgumentError("user id field contained invalid characters") + } + signingPriv, err := rsa.GenerateKey(config.Random(), defaultRSAKeyBits) + if err != nil { + return nil, err + } + encryptingPriv, err := rsa.GenerateKey(config.Random(), defaultRSAKeyBits) + if err != nil { + return nil, err + } + + e := &Entity{ + PrimaryKey: packet.NewRSAPublicKey(currentTime, &signingPriv.PublicKey), + PrivateKey: packet.NewRSAPrivateKey(currentTime, signingPriv), + Identities: make(map[string]*Identity), + } + isPrimaryId := true + e.Identities[uid.Id] = &Identity{ + Name: uid.Name, + UserId: uid, + SelfSignature: &packet.Signature{ + CreationTime: currentTime, + SigType: packet.SigTypePositiveCert, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: config.Hash(), + IsPrimaryId: &isPrimaryId, + FlagsValid: true, + FlagSign: true, + FlagCertify: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + + e.Subkeys = make([]Subkey, 1) + e.Subkeys[0] = Subkey{ + PublicKey: packet.NewRSAPublicKey(currentTime, &encryptingPriv.PublicKey), + PrivateKey: packet.NewRSAPrivateKey(currentTime, encryptingPriv), + Sig: &packet.Signature{ + CreationTime: currentTime, + SigType: packet.SigTypeSubkeyBinding, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: config.Hash(), + FlagsValid: true, + FlagEncryptStorage: true, + FlagEncryptCommunications: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + e.Subkeys[0].PublicKey.IsSubkey = true + e.Subkeys[0].PrivateKey.IsSubkey = true + + return e, nil +} + +// SerializePrivate serializes an Entity, including private key material, to +// the given Writer. For now, it must only be used on an Entity returned from +// NewEntity. +// If config is nil, sensible defaults will be used. +func (e *Entity) SerializePrivate(w io.Writer, config *packet.Config) (err error) { + err = e.PrivateKey.Serialize(w) + if err != nil { + return + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return + } + err = ident.SelfSignature.SignUserId(ident.UserId.Id, e.PrimaryKey, e.PrivateKey, config) + if err != nil { + return + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return + } + } + for _, subkey := range e.Subkeys { + err = subkey.PrivateKey.Serialize(w) + if err != nil { + return + } + err = subkey.Sig.SignKey(subkey.PublicKey, e.PrivateKey, config) + if err != nil { + return + } + err = subkey.Sig.Serialize(w) + if err != nil { + return + } + } + return nil +} + +// Serialize writes the public part of the given Entity to w. (No private +// key material will be output). +func (e *Entity) Serialize(w io.Writer) error { + err := e.PrimaryKey.Serialize(w) + if err != nil { + return err + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return err + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return err + } + for _, sig := range ident.Signatures { + err = sig.Serialize(w) + if err != nil { + return err + } + } + } + for _, subkey := range e.Subkeys { + err = subkey.PublicKey.Serialize(w) + if err != nil { + return err + } + err = subkey.Sig.Serialize(w) + if err != nil { + return err + } + } + return nil +} + +// SignIdentity adds a signature to e, from signer, attesting that identity is +// associated with e. The provided identity must already be an element of +// e.Identities and the private key of signer must have been decrypted if +// necessary. +// If config is nil, sensible defaults will be used. +func (e *Entity) SignIdentity(identity string, signer *Entity, config *packet.Config) error { + if signer.PrivateKey == nil { + return errors.InvalidArgumentError("signing Entity must have a private key") + } + if signer.PrivateKey.Encrypted { + return errors.InvalidArgumentError("signing Entity's private key must be decrypted") + } + ident, ok := e.Identities[identity] + if !ok { + return errors.InvalidArgumentError("given identity string not found in Entity") + } + + sig := &packet.Signature{ + SigType: packet.SigTypeGenericCert, + PubKeyAlgo: signer.PrivateKey.PubKeyAlgo, + Hash: config.Hash(), + CreationTime: config.Now(), + IssuerKeyId: &signer.PrivateKey.KeyId, + } + if err := sig.SignUserId(identity, e.PrimaryKey, signer.PrivateKey, config); err != nil { + return err + } + ident.Signatures = append(ident.Signatures, sig) + return nil +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/keys_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/keys_test.go new file mode 100644 index 00000000000..e0625cb424c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/keys_test.go @@ -0,0 +1,216 @@ +package openpgp + +import ( + "testing" + "time" + + "golang.org/x/crypto/openpgp/packet" +) + +func TestKeyExpiry(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(expiringKeyHex)) + entity := kring[0] + + const timeFormat = "2006-01-02" + time1, _ := time.Parse(timeFormat, "2013-07-01") + // The expiringKeyHex key is structured as: + // + // pub 1024R/5E237D8C created: 2013-07-01 expires: 2013-07-31 usage: SC + // sub 1024R/1ABB25A0 created: 2013-07-01 expires: 2013-07-08 usage: E + // sub 1024R/96A672F5 created: 2013-07-01 expires: 2013-07-31 usage: E + // + // So this should select the first, non-expired encryption key. + key, _ := entity.encryptionKey(time1) + if id := key.PublicKey.KeyIdShortString(); id != "1ABB25A0" { + t.Errorf("Expected key 1ABB25A0 at time %s, but got key %s", time1.Format(timeFormat), id) + } + + // Once the first encryption subkey has expired, the second should be + // selected. + time2, _ := time.Parse(timeFormat, "2013-07-09") + key, _ = entity.encryptionKey(time2) + if id := key.PublicKey.KeyIdShortString(); id != "96A672F5" { + t.Errorf("Expected key 96A672F5 at time %s, but got key %s", time2.Format(timeFormat), id) + } + + // Once all the keys have expired, nothing should be returned. + time3, _ := time.Parse(timeFormat, "2013-08-01") + if key, ok := entity.encryptionKey(time3); ok { + t.Errorf("Expected no key at time %s, but got key %s", time3.Format(timeFormat), key.PublicKey.KeyIdShortString()) + } +} + +// TestExternallyRevokableKey attempts to load and parse a key with a third party revocation permission. +func TestExternallyRevocableKey(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(subkeyUsageHex)) + + // The 0xA42704B92866382A key can be revoked by 0xBE3893CB843D0FE70C + // according to this signature that appears within the key: + // :signature packet: algo 1, keyid A42704B92866382A + // version 4, created 1396409682, md5len 0, sigclass 0x1f + // digest algo 2, begin of digest a9 84 + // hashed subpkt 2 len 4 (sig created 2014-04-02) + // hashed subpkt 12 len 22 (revocation key: c=80 a=1 f=CE094AA433F7040BB2DDF0BE3893CB843D0FE70C) + // hashed subpkt 7 len 1 (not revocable) + // subpkt 16 len 8 (issuer key ID A42704B92866382A) + // data: [1024 bits] + + id := uint64(0xA42704B92866382A) + keys := kring.KeysById(id) + if len(keys) != 1 { + t.Errorf("Expected to find key id %X, but got %d matches", id, len(keys)) + } +} + +func TestKeyRevocation(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(revokedKeyHex)) + + // revokedKeyHex contains these keys: + // pub 1024R/9A34F7C0 2014-03-25 [revoked: 2014-03-25] + // sub 1024R/1BA3CD60 2014-03-25 [revoked: 2014-03-25] + ids := []uint64{0xA401D9F09A34F7C0, 0x5CD3BE0A1BA3CD60} + + for _, id := range ids { + keys := kring.KeysById(id) + if len(keys) != 1 { + t.Errorf("Expected KeysById to find revoked key %X, but got %d matches", id, len(keys)) + } + keys = kring.KeysByIdUsage(id, 0) + if len(keys) != 0 { + t.Errorf("Expected KeysByIdUsage to filter out revoked key %X, but got %d matches", id, len(keys)) + } + } +} + +func TestSubkeyRevocation(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(revokedSubkeyHex)) + + // revokedSubkeyHex contains these keys: + // pub 1024R/4EF7E4BECCDE97F0 2014-03-25 + // sub 1024R/D63636E2B96AE423 2014-03-25 + // sub 1024D/DBCE4EE19529437F 2014-03-25 + // sub 1024R/677815E371C2FD23 2014-03-25 [revoked: 2014-03-25] + validKeys := []uint64{0x4EF7E4BECCDE97F0, 0xD63636E2B96AE423, 0xDBCE4EE19529437F} + revokedKey := uint64(0x677815E371C2FD23) + + for _, id := range validKeys { + keys := kring.KeysById(id) + if len(keys) != 1 { + t.Errorf("Expected KeysById to find key %X, but got %d matches", id, len(keys)) + } + keys = kring.KeysByIdUsage(id, 0) + if len(keys) != 1 { + t.Errorf("Expected KeysByIdUsage to find key %X, but got %d matches", id, len(keys)) + } + } + + keys := kring.KeysById(revokedKey) + if len(keys) != 1 { + t.Errorf("Expected KeysById to find key %X, but got %d matches", revokedKey, len(keys)) + } + + keys = kring.KeysByIdUsage(revokedKey, 0) + if len(keys) != 0 { + t.Errorf("Expected KeysByIdUsage to filter out revoked key %X, but got %d matches", revokedKey, len(keys)) + } +} + +func TestKeyUsage(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(subkeyUsageHex)) + + // subkeyUsageHex contains these keys: + // pub 1024R/2866382A created: 2014-04-01 expires: never usage: SC + // sub 1024R/936C9153 created: 2014-04-01 expires: never usage: E + // sub 1024R/64D5F5BB created: 2014-04-02 expires: never usage: E + // sub 1024D/BC0BA992 created: 2014-04-02 expires: never usage: S + certifiers := []uint64{0xA42704B92866382A} + signers := []uint64{0xA42704B92866382A, 0x42CE2C64BC0BA992} + encrypters := []uint64{0x09C0C7D9936C9153, 0xC104E98664D5F5BB} + + for _, id := range certifiers { + keys := kring.KeysByIdUsage(id, packet.KeyFlagCertify) + if len(keys) == 1 { + if keys[0].PublicKey.KeyId != id { + t.Errorf("Expected to find certifier key id %X, but got %X", id, keys[0].PublicKey.KeyId) + } + } else { + t.Errorf("Expected one match for certifier key id %X, but got %d matches", id, len(keys)) + } + } + + for _, id := range signers { + keys := kring.KeysByIdUsage(id, packet.KeyFlagSign) + if len(keys) == 1 { + if keys[0].PublicKey.KeyId != id { + t.Errorf("Expected to find signing key id %X, but got %X", id, keys[0].PublicKey.KeyId) + } + } else { + t.Errorf("Expected one match for signing key id %X, but got %d matches", id, len(keys)) + } + + // This keyring contains no encryption keys that are also good for signing. + keys = kring.KeysByIdUsage(id, packet.KeyFlagEncryptStorage|packet.KeyFlagEncryptCommunications) + if len(keys) != 0 { + t.Errorf("Unexpected match for encryption key id %X", id) + } + } + + for _, id := range encrypters { + keys := kring.KeysByIdUsage(id, packet.KeyFlagEncryptStorage|packet.KeyFlagEncryptCommunications) + if len(keys) == 1 { + if keys[0].PublicKey.KeyId != id { + t.Errorf("Expected to find encryption key id %X, but got %X", id, keys[0].PublicKey.KeyId) + } + } else { + t.Errorf("Expected one match for encryption key id %X, but got %d matches", id, len(keys)) + } + + // This keyring contains no encryption keys that are also good for signing. + keys = kring.KeysByIdUsage(id, packet.KeyFlagSign) + if len(keys) != 0 { + t.Errorf("Unexpected match for signing key id %X", id) + } + } +} + +func TestIdVerification(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + if err != nil { + t.Fatal(err) + } + if err := kring[1].PrivateKey.Decrypt([]byte("passphrase")); err != nil { + t.Fatal(err) + } + + const identity = "Test Key 1 (RSA)" + if err := kring[0].SignIdentity(identity, kring[1], nil); err != nil { + t.Fatal(err) + } + + ident, ok := kring[0].Identities[identity] + if !ok { + t.Fatal("identity missing from key after signing") + } + + checked := false + for _, sig := range ident.Signatures { + if sig.IssuerKeyId == nil || *sig.IssuerKeyId != kring[1].PrimaryKey.KeyId { + continue + } + + if err := kring[1].PrimaryKey.VerifyUserIdSignature(identity, kring[0].PrimaryKey, sig); err != nil { + t.Fatalf("error verifying new identity signature: %s", err) + } + checked = true + break + } + + if !checked { + t.Fatal("didn't find identity signature in Entity") + } +} + +const expiringKeyHex = "988d0451d1ec5d010400ba3385721f2dc3f4ab096b2ee867ab77213f0a27a8538441c35d2fa225b08798a1439a66a5150e6bdc3f40f5d28d588c712394c632b6299f77db8c0d48d37903fb72ebd794d61be6aa774688839e5fdecfe06b2684cc115d240c98c66cb1ef22ae84e3aa0c2b0c28665c1e7d4d044e7f270706193f5223c8d44e0d70b7b8da830011010001b40f4578706972792074657374206b657988be041301020028050251d1ec5d021b03050900278d00060b090807030206150802090a0b0416020301021e01021780000a091072589ad75e237d8c033503fd10506d72837834eb7f994117740723adc39227104b0d326a1161871c0b415d25b4aedef946ca77ea4c05af9c22b32cf98be86ab890111fced1ee3f75e87b7cc3c00dc63bbc85dfab91c0dc2ad9de2c4d13a34659333a85c6acc1a669c5e1d6cecb0cf1e56c10e72d855ae177ddc9e766f9b2dda57ccbb75f57156438bbdb4e42b88d0451d1ec5d0104009c64906559866c5cb61578f5846a94fcee142a489c9b41e67b12bb54cfe86eb9bc8566460f9a720cb00d6526fbccfd4f552071a8e3f7744b1882d01036d811ee5a3fb91a1c568055758f43ba5d2c6a9676b012f3a1a89e47bbf624f1ad571b208f3cc6224eb378f1645dd3d47584463f9eadeacfd1ce6f813064fbfdcc4b5a53001101000188a504180102000f021b0c050251d1f06b050900093e89000a091072589ad75e237d8c20e00400ab8310a41461425b37889c4da28129b5fae6084fafbc0a47dd1adc74a264c6e9c9cc125f40462ee1433072a58384daef88c961c390ed06426a81b464a53194c4e291ddd7e2e2ba3efced01537d713bd111f48437bde2363446200995e8e0d4e528dda377fd1e8f8ede9c8e2198b393bd86852ce7457a7e3daf74d510461a5b77b88d0451d1ece8010400b3a519f83ab0010307e83bca895170acce8964a044190a2b368892f7a244758d9fc193482648acb1fb9780d28cc22d171931f38bb40279389fc9bf2110876d4f3db4fcfb13f22f7083877fe56592b3b65251312c36f83ffcb6d313c6a17f197dd471f0712aad15a8537b435a92471ba2e5b0c72a6c72536c3b567c558d7b6051001101000188a504180102000f021b0c050251d1f07b050900279091000a091072589ad75e237d8ce69e03fe286026afacf7c97ee20673864d4459a2240b5655219950643c7dba0ac384b1d4359c67805b21d98211f7b09c2a0ccf6410c8c04d4ff4a51293725d8d6570d9d8bb0e10c07d22357caeb49626df99c180be02d77d1fe8ed25e7a54481237646083a9f89a11566cd20b9e995b1487c5f9e02aeb434f3a1897cd416dd0a87861838da3e9e" +const subkeyUsageHex = "988d04533a52bc010400d26af43085558f65b9e7dbc90cb9238015259aed5e954637adcfa2181548b2d0b60c65f1f42ec5081cbf1bc0a8aa4900acfb77070837c58f26012fbce297d70afe96e759ad63531f0037538e70dbf8e384569b9720d99d8eb39d8d0a2947233ed242436cb6ac7dfe74123354b3d0119b5c235d3dd9c9d6c004f8ffaf67ad8583001101000188b7041f010200210502533b8552170c8001ce094aa433f7040bb2ddf0be3893cb843d0fe70c020700000a0910a42704b92866382aa98404009d63d916a27543da4221c60087c33f1c44bec9998c5438018ed370cca4962876c748e94b73eb39c58eb698063f3fd6346d58dd2a11c0247934c4a9d71f24754f7468f96fb24c3e791dd2392b62f626148ad724189498cbf993db2df7c0cdc2d677c35da0f16cb16c9ce7c33b4de65a4a91b1d21a130ae9cc26067718910ef8e2b417556d627261203c756d627261407379642e65642e61753e88b80413010200220502533a52bc021b03060b090807030206150802090a0b0416020301021e01021780000a0910a42704b92866382a47840400c0c2bd04f5fca586de408b395b3c280a278259c93eaaa8b79a53b97003f8ed502a8a00446dd9947fb462677e4fcac0dac2f0701847d15130aadb6cd9e0705ea0cf5f92f129136c7be21a718d46c8e641eb7f044f2adae573e11ae423a0a9ca51324f03a8a2f34b91fa40c3cc764bee4dccadedb54c768ba0469b683ea53f1c29b88d04533a52bc01040099c92a5d6f8b744224da27bc2369127c35269b58bec179de6bbc038f749344222f85a31933224f26b70243c4e4b2d242f0c4777eaef7b5502f9dad6d8bf3aaeb471210674b74de2d7078af497d55f5cdad97c7bedfbc1b41e8065a97c9c3d344b21fc81d27723af8e374bc595da26ea242dccb6ae497be26eea57e563ed517e90011010001889f0418010200090502533a52bc021b0c000a0910a42704b92866382afa1403ff70284c2de8a043ff51d8d29772602fa98009b7861c540535f874f2c230af8caf5638151a636b21f8255003997ccd29747fdd06777bb24f9593bd7d98a3e887689bf902f999915fcc94625ae487e5d13e6616f89090ebc4fdc7eb5cad8943e4056995bb61c6af37f8043016876a958ec7ebf39c43d20d53b7f546cfa83e8d2604b88d04533b8283010400c0b529316dbdf58b4c54461e7e669dc11c09eb7f73819f178ccd4177b9182b91d138605fcf1e463262fabefa73f94a52b5e15d1904635541c7ea540f07050ce0fb51b73e6f88644cec86e91107c957a114f69554548a85295d2b70bd0b203992f76eb5d493d86d9eabcaa7ef3fc7db7e458438db3fcdb0ca1cc97c638439a9170011010001889f0418010200090502533b8283021b0c000a0910a42704b92866382adc6d0400cfff6258485a21675adb7a811c3e19ebca18851533f75a7ba317950b9997fda8d1a4c8c76505c08c04b6c2cc31dc704d33da36a21273f2b388a1a706f7c3378b66d887197a525936ed9a69acb57fe7f718133da85ec742001c5d1864e9c6c8ea1b94f1c3759cebfd93b18606066c063a63be86085b7e37bdbc65f9a915bf084bb901a204533b85cd110400aed3d2c52af2b38b5b67904b0ef73d6dd7aef86adb770e2b153cd22489654dcc91730892087bb9856ae2d9f7ed1eb48f214243fe86bfe87b349ebd7c30e630e49c07b21fdabf78b7a95c8b7f969e97e3d33f2e074c63552ba64a2ded7badc05ce0ea2be6d53485f6900c7860c7aa76560376ce963d7271b9b54638a4028b573f00a0d8854bfcdb04986141568046202192263b9b67350400aaa1049dbc7943141ef590a70dcb028d730371d92ea4863de715f7f0f16d168bd3dc266c2450457d46dcbbf0b071547e5fbee7700a820c3750b236335d8d5848adb3c0da010e998908dfd93d961480084f3aea20b247034f8988eccb5546efaa35a92d0451df3aaf1aee5aa36a4c4d462c760ecd9cebcabfbe1412b1f21450f203fd126687cd486496e971a87fd9e1a8a765fe654baa219a6871ab97768596ab05c26c1aeea8f1a2c72395a58dbc12ef9640d2b95784e974a4d2d5a9b17c25fedacfe551bda52602de8f6d2e48443f5dd1a2a2a8e6a5e70ecdb88cd6e766ad9745c7ee91d78cc55c3d06536b49c3fee6c3d0b6ff0fb2bf13a314f57c953b8f4d93bf88e70418010200090502533b85cd021b0200520910a42704b92866382a47200419110200060502533b85cd000a091042ce2c64bc0ba99214b2009e26b26852c8b13b10c35768e40e78fbbb48bd084100a0c79d9ea0844fa5853dd3c85ff3ecae6f2c9dd6c557aa04008bbbc964cd65b9b8299d4ebf31f41cc7264b8cf33a00e82c5af022331fac79efc9563a822497ba012953cefe2629f1242fcdcb911dbb2315985bab060bfd58261ace3c654bdbbe2e8ed27a46e836490145c86dc7bae15c011f7e1ffc33730109b9338cd9f483e7cef3d2f396aab5bd80efb6646d7e778270ee99d934d187dd98" +const revokedKeyHex = "988d045331ce82010400c4fdf7b40a5477f206e6ee278eaef888ca73bf9128a9eef9f2f1ddb8b7b71a4c07cfa241f028a04edb405e4d916c61d6beabc333813dc7b484d2b3c52ee233c6a79b1eea4e9cc51596ba9cd5ac5aeb9df62d86ea051055b79d03f8a4fa9f38386f5bd17529138f3325d46801514ea9047977e0829ed728e68636802796801be10011010001889f04200102000905025331d0e3021d03000a0910a401d9f09a34f7c042aa040086631196405b7e6af71026b88e98012eab44aa9849f6ef3fa930c7c9f23deaedba9db1538830f8652fb7648ec3fcade8dbcbf9eaf428e83c6cbcc272201bfe2fbb90d41963397a7c0637a1a9d9448ce695d9790db2dc95433ad7be19eb3de72dacf1d6db82c3644c13eae2a3d072b99bb341debba012c5ce4006a7d34a1f4b94b444526567205265766f6b657220283c52656727732022424d204261726973746122204b657920262530305c303e5c29203c72656740626d626172697374612e636f2e61753e88b704130102002205025331ce82021b03060b090807030206150802090a0b0416020301021e01021780000a0910a401d9f09a34f7c0019c03f75edfbeb6a73e7225ad3cc52724e2872e04260d7daf0d693c170d8c4b243b8767bc7785763533febc62ec2600c30603c433c095453ede59ff2fcabeb84ce32e0ed9d5cf15ffcbc816202b64370d4d77c1e9077d74e94a16fb4fa2e5bec23a56d7a73cf275f91691ae1801a976fcde09e981a2f6327ac27ea1fecf3185df0d56889c04100102000605025331cfb5000a0910fe9645554e8266b64b4303fc084075396674fb6f778d302ac07cef6bc0b5d07b66b2004c44aef711cbac79617ef06d836b4957522d8772dd94bf41a2f4ac8b1ee6d70c57503f837445a74765a076d07b829b8111fc2a918423ddb817ead7ca2a613ef0bfb9c6b3562aec6c3cf3c75ef3031d81d95f6563e4cdcc9960bcb386c5d757b104fcca5fe11fc709df884604101102000605025331cfe7000a09107b15a67f0b3ddc0317f6009e360beea58f29c1d963a22b962b80788c3fa6c84e009d148cfde6b351469b8eae91187eff07ad9d08fcaab88d045331ce820104009f25e20a42b904f3fa555530fe5c46737cf7bd076c35a2a0d22b11f7e0b61a69320b768f4a80fe13980ce380d1cfc4a0cd8fbe2d2e2ef85416668b77208baa65bf973fe8e500e78cc310d7c8705cdb34328bf80e24f0385fce5845c33bc7943cf6b11b02348a23da0bf6428e57c05135f2dc6bd7c1ce325d666d5a5fd2fd5e410011010001889f04180102000905025331ce82021b0c000a0910a401d9f09a34f7c0418003fe34feafcbeaef348a800a0d908a7a6809cc7304017d820f70f0474d5e23cb17e38b67dc6dca282c6ca00961f4ec9edf2738d0f087b1d81e4871ef08e1798010863afb4eac4c44a376cb343be929c5be66a78cfd4456ae9ec6a99d97f4e1c3ff3583351db2147a65c0acef5c003fb544ab3a2e2dc4d43646f58b811a6c3a369d1f" +const revokedSubkeyHex = "988d04533121f6010400aefc803a3e4bb1a61c86e8a86d2726c6a43e0079e9f2713f1fa017e9854c83877f4aced8e331d675c67ea83ddab80aacbfa0b9040bb12d96f5a3d6be09455e2a76546cbd21677537db941cab710216b6d24ec277ee0bd65b910f416737ed120f6b93a9d3b306245c8cfd8394606fdb462e5cf43c551438d2864506c63367fc890011010001b41d416c696365203c616c69636540626d626172697374612e636f2e61753e88bb041301020025021b03060b090807030206150802090a0b0416020301021e01021780050253312798021901000a09104ef7e4beccde97f015a803ff5448437780f63263b0df8442a995e7f76c221351a51edd06f2063d8166cf3157aada4923dfc44aa0f2a6a4da5cf83b7fe722ba8ab416c976e77c6b5682e7f1069026673bd0de56ba06fd5d7a9f177607f277d9b55ff940a638c3e68525c67517e2b3d976899b93ca267f705b3e5efad7d61220e96b618a4497eab8d04403d23f8846041011020006050253312910000a09107b15a67f0b3ddc03d96e009f50b6365d86c4be5d5e9d0ea42d5e56f5794c617700a0ab274e19c2827780016d23417ce89e0a2c0d987d889c04100102000605025331cf7a000a0910a401d9f09a34f7c0ee970400aca292f213041c9f3b3fc49148cbda9d84afee6183c8dd6c5ff2600b29482db5fecd4303797be1ee6d544a20a858080fec43412061c9a71fae4039fd58013b4ae341273e6c66ad4c7cdd9e68245bedb260562e7b166f2461a1032f2b38c0e0e5715fb3d1656979e052b55ca827a76f872b78a9fdae64bc298170bfcebedc1271b41a416c696365203c616c696365407379646973702e6f722e61753e88b804130102002205025331278b021b03060b090807030206150802090a0b0416020301021e01021780000a09104ef7e4beccde97f06a7003fa03c3af68d272ebc1fa08aa72a03b02189c26496a2833d90450801c4e42c5b5f51ad96ce2d2c9cef4b7c02a6a2fcf1412d6a2d486098eb762f5010a201819c17fd2888aec8eda20c65a3b75744de7ee5cc8ac7bfc470cbe3cb982720405a27a3c6a8c229cfe36905f881b02ed5680f6a8f05866efb9d6c5844897e631deb949ca8846041011020006050253312910000a09107b15a67f0b3ddc0347bc009f7fa35db59147469eb6f2c5aaf6428accb138b22800a0caa2f5f0874bacc5909c652a57a31beda65eddd5889c04100102000605025331cf7a000a0910a401d9f09a34f7c0316403ff46f2a5c101256627f16384d34a38fb47a6c88ba60506843e532d91614339fccae5f884a5741e7582ffaf292ba38ee10a270a05f139bde3814b6a077e8cd2db0f105ebea2a83af70d385f13b507fac2ad93ff79d84950328bb86f3074745a8b7f9b64990fb142e2a12976e27e8d09a28dc5621f957ac49091116da410ac3cbde1b88d04533121f6010400cbd785b56905e4192e2fb62a720727d43c4fa487821203cf72138b884b78b701093243e1d8c92a0248a6c0203a5a88693da34af357499abacaf4b3309c640797d03093870a323b4b6f37865f6eaa2838148a67df4735d43a90ca87942554cdf1c4a751b1e75f9fd4ce4e97e278d6c1c7ed59d33441df7d084f3f02beb68896c70011010001889f0418010200090502533121f6021b0c000a09104ef7e4beccde97f0b98b03fc0a5ccf6a372995835a2f5da33b282a7d612c0ab2a97f59cf9fff73e9110981aac2858c41399afa29624a7fd8a0add11654e3d882c0fd199e161bdad65e5e2548f7b68a437ea64293db1246e3011cbb94dc1bcdeaf0f2539bd88ff16d95547144d97cead6a8c5927660a91e6db0d16eb36b7b49a3525b54d1644e65599b032b7eb901a204533127a0110400bd3edaa09eff9809c4edc2c2a0ebe52e53c50a19c1e49ab78e6167bf61473bb08f2050d78a5cbbc6ed66aff7b42cd503f16b4a0b99fa1609681fca9b7ce2bbb1a5b3864d6cdda4d7ef7849d156d534dea30fb0efb9e4cf8959a2b2ce623905882d5430b995a15c3b9fe92906086788b891002924f94abe139b42cbbfaaabe42f00a0b65dc1a1ad27d798adbcb5b5ad02d2688c89477b03ff4eebb6f7b15a73b96a96bed201c0e5e4ea27e4c6e2dd1005b94d4b90137a5b1cf5e01c6226c070c4cc999938101578877ee76d296b9aab8246d57049caacf489e80a3f40589cade790a020b1ac146d6f7a6241184b8c7fcde680eae3188f5dcbe846d7f7bdad34f6fcfca08413e19c1d5df83fc7c7c627d493492e009c2f52a80400a2fe82de87136fd2e8845888c4431b032ba29d9a29a804277e31002a8201fb8591a3e55c7a0d0881496caf8b9fb07544a5a4879291d0dc026a0ea9e5bd88eb4aa4947bbd694b25012e208a250d65ddc6f1eea59d3aed3b4ec15fcab85e2afaa23a40ab1ef9ce3e11e1bc1c34a0e758e7aa64deb8739276df0af7d4121f834a9b88e70418010200090502533127a0021b02005209104ef7e4beccde97f047200419110200060502533127a0000a0910dbce4ee19529437fe045009c0b32f5ead48ee8a7e98fac0dea3d3e6c0e2c552500a0ad71fadc5007cfaf842d9b7db3335a8cdad15d3d1a6404009b08e2c68fe8f3b45c1bb72a4b3278cdf3012aa0f229883ad74aa1f6000bb90b18301b2f85372ca5d6b9bf478d235b733b1b197d19ccca48e9daf8e890cb64546b4ce1b178faccfff07003c172a2d4f5ebaba9f57153955f3f61a9b80a4f5cb959908f8b211b03b7026a8a82fc612bfedd3794969bcf458c4ce92be215a1176ab88d045331d144010400a5063000c5aaf34953c1aa3bfc95045b3aab9882b9a8027fecfe2142dc6b47ba8aca667399990244d513dd0504716908c17d92c65e74219e004f7b83fc125e575dd58efec3ab6dd22e3580106998523dea42ec75bf9aa111734c82df54630bebdff20fe981cfc36c76f865eb1c2fb62c9e85bc3a6e5015a361a2eb1c8431578d0011010001889f04280102000905025331d433021d03000a09104ef7e4beccde97f02e5503ff5e0630d1b65291f4882b6d40a29da4616bb5088717d469fbcc3648b8276de04a04988b1f1b9f3e18f52265c1f8b6c85861691c1a6b8a3a25a1809a0b32ad330aec5667cb4262f4450649184e8113849b05e5ad06a316ea80c001e8e71838190339a6e48bbde30647bcf245134b9a97fa875c1d83a9862cae87ffd7e2c4ce3a1b89013d04180102000905025331d144021b0200a809104ef7e4beccde97f09d2004190102000605025331d144000a0910677815e371c2fd23522203fe22ab62b8e7a151383cea3edd3a12995693911426f8ccf125e1f6426388c0010f88d9ca7da2224aee8d1c12135998640c5e1813d55a93df472faae75bef858457248db41b4505827590aeccf6f9eb646da7f980655dd3050c6897feddddaca90676dee856d66db8923477d251712bb9b3186b4d0114daf7d6b59272b53218dd1da94a03ff64006fcbe71211e5daecd9961fba66cdb6de3f914882c58ba5beddeba7dcb950c1156d7fba18c19ea880dccc800eae335deec34e3b84ac75ffa24864f782f87815cda1c0f634b3dd2fa67cea30811d21723d21d9551fa12ccbcfa62b6d3a15d01307b99925707992556d50065505b090aadb8579083a20fe65bd2a270da9b011" diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/compressed.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/compressed.go new file mode 100644 index 00000000000..e8f0b5caa7d --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/compressed.go @@ -0,0 +1,123 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "compress/bzip2" + "compress/flate" + "compress/zlib" + "golang.org/x/crypto/openpgp/errors" + "io" + "strconv" +) + +// Compressed represents a compressed OpenPGP packet. The decompressed contents +// will contain more OpenPGP packets. See RFC 4880, section 5.6. +type Compressed struct { + Body io.Reader +} + +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression +) + +// CompressionConfig contains compressor configuration settings. +type CompressionConfig struct { + // Level is the compression level to use. It must be set to + // between -1 and 9, with -1 causing the compressor to use the + // default compression level, 0 causing the compressor to use + // no compression and 1 to 9 representing increasing (better, + // slower) compression levels. If Level is less than -1 or + // more then 9, a non-nil error will be returned during + // encryption. See the constants above for convenient common + // settings for Level. + Level int +} + +func (c *Compressed) parse(r io.Reader) error { + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + + switch buf[0] { + case 1: + c.Body = flate.NewReader(r) + case 2: + c.Body, err = zlib.NewReader(r) + case 3: + c.Body = bzip2.NewReader(r) + default: + err = errors.UnsupportedError("unknown compression algorithm: " + strconv.Itoa(int(buf[0]))) + } + + return err +} + +// compressedWriterCloser represents the serialized compression stream +// header and the compressor. Its Close() method ensures that both the +// compressor and serialized stream header are closed. Its Write() +// method writes to the compressor. +type compressedWriteCloser struct { + sh io.Closer // Stream Header + c io.WriteCloser // Compressor +} + +func (cwc compressedWriteCloser) Write(p []byte) (int, error) { + return cwc.c.Write(p) +} + +func (cwc compressedWriteCloser) Close() (err error) { + err = cwc.c.Close() + if err != nil { + return err + } + + return cwc.sh.Close() +} + +// SerializeCompressed serializes a compressed data packet to w and +// returns a WriteCloser to which the literal data packets themselves +// can be written and which MUST be closed on completion. If cc is +// nil, sensible defaults will be used to configure the compression +// algorithm. +func SerializeCompressed(w io.WriteCloser, algo CompressionAlgo, cc *CompressionConfig) (literaldata io.WriteCloser, err error) { + compressed, err := serializeStreamHeader(w, packetTypeCompressed) + if err != nil { + return + } + + _, err = compressed.Write([]byte{uint8(algo)}) + if err != nil { + return + } + + level := DefaultCompression + if cc != nil { + level = cc.Level + } + + var compressor io.WriteCloser + switch algo { + case CompressionZIP: + compressor, err = flate.NewWriter(compressed, level) + case CompressionZLIB: + compressor, err = zlib.NewWriterLevel(compressed, level) + default: + s := strconv.Itoa(int(algo)) + err = errors.UnsupportedError("Unsupported compression algorithm: " + s) + } + if err != nil { + return + } + + literaldata = compressedWriteCloser{compressed, compressor} + + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/compressed_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/compressed_test.go new file mode 100644 index 00000000000..cb2d70bd411 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/compressed_test.go @@ -0,0 +1,41 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "io" + "io/ioutil" + "testing" +) + +func TestCompressed(t *testing.T) { + packet, err := Read(readerFromHex(compressedHex)) + if err != nil { + t.Errorf("failed to read Compressed: %s", err) + return + } + + c, ok := packet.(*Compressed) + if !ok { + t.Error("didn't find Compressed packet") + return + } + + contents, err := ioutil.ReadAll(c.Body) + if err != nil && err != io.EOF { + t.Error(err) + return + } + + expected, _ := hex.DecodeString(compressedExpectedHex) + if !bytes.Equal(expected, contents) { + t.Errorf("got:%x want:%x", contents, expected) + } +} + +const compressedHex = "a3013b2d90c4e02b72e25f727e5e496a5e49b11e1700" +const compressedExpectedHex = "cb1062004d14c8fe636f6e74656e74732e0a" diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/config.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/config.go new file mode 100644 index 00000000000..d977cde63f1 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/config.go @@ -0,0 +1,88 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/rand" + "io" + "time" +) + +// Config collects a number of parameters along with sensible defaults. +// A nil *Config is valid and results in all default values. +type Config struct { + // Rand provides the source of entropy. + // If nil, the crypto/rand Reader is used. + Rand io.Reader + // DefaultHash is the default hash function to be used. + // If zero, SHA-256 is used. + DefaultHash crypto.Hash + // DefaultCipher is the cipher to be used. + // If zero, AES-128 is used. + DefaultCipher CipherFunction + // Time returns the current time as the number of seconds since the + // epoch. If Time is nil, time.Now is used. + Time func() time.Time + // DefaultCompressionAlgo is the compression algorithm to be + // applied to the plaintext before encryption. If zero, no + // compression is done. + DefaultCompressionAlgo CompressionAlgo + // CompressionConfig configures the compression settings. + CompressionConfig *CompressionConfig + // S2KCount is only used for symmetric encryption. It + // determines the strength of the passphrase stretching when + // the said passphrase is hashed to produce a key. S2KCount + // should be between 1024 and 65011712, inclusive. If Config + // is nil or S2KCount is 0, the value 65536 used. Not all + // values in the above range can be represented. S2KCount will + // be rounded up to the next representable value if it cannot + // be encoded exactly. When set, it is strongly encrouraged to + // use a value that is at least 65536. See RFC 4880 Section + // 3.7.1.3. + S2KCount int +} + +func (c *Config) Random() io.Reader { + if c == nil || c.Rand == nil { + return rand.Reader + } + return c.Rand +} + +func (c *Config) Hash() crypto.Hash { + if c == nil || uint(c.DefaultHash) == 0 { + return crypto.SHA256 + } + return c.DefaultHash +} + +func (c *Config) Cipher() CipherFunction { + if c == nil || uint8(c.DefaultCipher) == 0 { + return CipherAES128 + } + return c.DefaultCipher +} + +func (c *Config) Now() time.Time { + if c == nil || c.Time == nil { + return time.Now() + } + return c.Time() +} + +func (c *Config) Compression() CompressionAlgo { + if c == nil { + return CompressionNone + } + return c.DefaultCompressionAlgo +} + +func (c *Config) PasswordHashIterations() int { + if c == nil || c.S2KCount == 0 { + return 0 + } + return c.S2KCount +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/encrypted_key.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/encrypted_key.go new file mode 100644 index 00000000000..b43522cd50d --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/encrypted_key.go @@ -0,0 +1,168 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/rsa" + "encoding/binary" + "golang.org/x/crypto/openpgp/elgamal" + "golang.org/x/crypto/openpgp/errors" + "io" + "math/big" + "strconv" +) + +const encryptedKeyVersion = 3 + +// EncryptedKey represents a public-key encrypted session key. See RFC 4880, +// section 5.1. +type EncryptedKey struct { + KeyId uint64 + Algo PublicKeyAlgorithm + CipherFunc CipherFunction // only valid after a successful Decrypt + Key []byte // only valid after a successful Decrypt + + encryptedMPI1, encryptedMPI2 []byte +} + +func (e *EncryptedKey) parse(r io.Reader) (err error) { + var buf [10]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != encryptedKeyVersion { + return errors.UnsupportedError("unknown EncryptedKey version " + strconv.Itoa(int(buf[0]))) + } + e.KeyId = binary.BigEndian.Uint64(buf[1:9]) + e.Algo = PublicKeyAlgorithm(buf[9]) + switch e.Algo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + e.encryptedMPI1, _, err = readMPI(r) + case PubKeyAlgoElGamal: + e.encryptedMPI1, _, err = readMPI(r) + if err != nil { + return + } + e.encryptedMPI2, _, err = readMPI(r) + } + _, err = consumeAll(r) + return +} + +func checksumKeyMaterial(key []byte) uint16 { + var checksum uint16 + for _, v := range key { + checksum += uint16(v) + } + return checksum +} + +// Decrypt decrypts an encrypted session key with the given private key. The +// private key must have been decrypted first. +// If config is nil, sensible defaults will be used. +func (e *EncryptedKey) Decrypt(priv *PrivateKey, config *Config) error { + var err error + var b []byte + + // TODO(agl): use session key decryption routines here to avoid + // padding oracle attacks. + switch priv.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + b, err = rsa.DecryptPKCS1v15(config.Random(), priv.PrivateKey.(*rsa.PrivateKey), e.encryptedMPI1) + case PubKeyAlgoElGamal: + c1 := new(big.Int).SetBytes(e.encryptedMPI1) + c2 := new(big.Int).SetBytes(e.encryptedMPI2) + b, err = elgamal.Decrypt(priv.PrivateKey.(*elgamal.PrivateKey), c1, c2) + default: + err = errors.InvalidArgumentError("cannot decrypted encrypted session key with private key of type " + strconv.Itoa(int(priv.PubKeyAlgo))) + } + + if err != nil { + return err + } + + e.CipherFunc = CipherFunction(b[0]) + e.Key = b[1 : len(b)-2] + expectedChecksum := uint16(b[len(b)-2])<<8 | uint16(b[len(b)-1]) + checksum := checksumKeyMaterial(e.Key) + if checksum != expectedChecksum { + return errors.StructuralError("EncryptedKey checksum incorrect") + } + + return nil +} + +// SerializeEncryptedKey serializes an encrypted key packet to w that contains +// key, encrypted to pub. +// If config is nil, sensible defaults will be used. +func SerializeEncryptedKey(w io.Writer, pub *PublicKey, cipherFunc CipherFunction, key []byte, config *Config) error { + var buf [10]byte + buf[0] = encryptedKeyVersion + binary.BigEndian.PutUint64(buf[1:9], pub.KeyId) + buf[9] = byte(pub.PubKeyAlgo) + + keyBlock := make([]byte, 1 /* cipher type */ +len(key)+2 /* checksum */) + keyBlock[0] = byte(cipherFunc) + copy(keyBlock[1:], key) + checksum := checksumKeyMaterial(key) + keyBlock[1+len(key)] = byte(checksum >> 8) + keyBlock[1+len(key)+1] = byte(checksum) + + switch pub.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + return serializeEncryptedKeyRSA(w, config.Random(), buf, pub.PublicKey.(*rsa.PublicKey), keyBlock) + case PubKeyAlgoElGamal: + return serializeEncryptedKeyElGamal(w, config.Random(), buf, pub.PublicKey.(*elgamal.PublicKey), keyBlock) + case PubKeyAlgoDSA, PubKeyAlgoRSASignOnly: + return errors.InvalidArgumentError("cannot encrypt to public key of type " + strconv.Itoa(int(pub.PubKeyAlgo))) + } + + return errors.UnsupportedError("encrypting a key to public key of type " + strconv.Itoa(int(pub.PubKeyAlgo))) +} + +func serializeEncryptedKeyRSA(w io.Writer, rand io.Reader, header [10]byte, pub *rsa.PublicKey, keyBlock []byte) error { + cipherText, err := rsa.EncryptPKCS1v15(rand, pub, keyBlock) + if err != nil { + return errors.InvalidArgumentError("RSA encryption failed: " + err.Error()) + } + + packetLen := 10 /* header length */ + 2 /* mpi size */ + len(cipherText) + + err = serializeHeader(w, packetTypeEncryptedKey, packetLen) + if err != nil { + return err + } + _, err = w.Write(header[:]) + if err != nil { + return err + } + return writeMPI(w, 8*uint16(len(cipherText)), cipherText) +} + +func serializeEncryptedKeyElGamal(w io.Writer, rand io.Reader, header [10]byte, pub *elgamal.PublicKey, keyBlock []byte) error { + c1, c2, err := elgamal.Encrypt(rand, pub, keyBlock) + if err != nil { + return errors.InvalidArgumentError("ElGamal encryption failed: " + err.Error()) + } + + packetLen := 10 /* header length */ + packetLen += 2 /* mpi size */ + (c1.BitLen()+7)/8 + packetLen += 2 /* mpi size */ + (c2.BitLen()+7)/8 + + err = serializeHeader(w, packetTypeEncryptedKey, packetLen) + if err != nil { + return err + } + _, err = w.Write(header[:]) + if err != nil { + return err + } + err = writeBig(w, c1) + if err != nil { + return err + } + return writeBig(w, c2) +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/encrypted_key_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/encrypted_key_test.go new file mode 100644 index 00000000000..0a8dcc6d857 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/encrypted_key_test.go @@ -0,0 +1,125 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/rsa" + "fmt" + "math/big" + "testing" +) + +func bigFromBase10(s string) *big.Int { + b, ok := new(big.Int).SetString(s, 10) + if !ok { + panic("bigFromBase10 failed") + } + return b +} + +var encryptedKeyPub = rsa.PublicKey{ + E: 65537, + N: bigFromBase10("115804063926007623305902631768113868327816898845124614648849934718568541074358183759250136204762053879858102352159854352727097033322663029387610959884180306668628526686121021235757016368038585212410610742029286439607686208110250133174279811431933746643015923132833417396844716207301518956640020862630546868823"), +} + +var encryptedKeyRSAPriv = &rsa.PrivateKey{ + PublicKey: encryptedKeyPub, + D: bigFromBase10("32355588668219869544751561565313228297765464314098552250409557267371233892496951383426602439009993875125222579159850054973310859166139474359774543943714622292329487391199285040721944491839695981199720170366763547754915493640685849961780092241140181198779299712578774460837139360803883139311171713302987058393"), +} + +var encryptedKeyPriv = &PrivateKey{ + PublicKey: PublicKey{ + PubKeyAlgo: PubKeyAlgoRSA, + }, + PrivateKey: encryptedKeyRSAPriv, +} + +func TestDecryptingEncryptedKey(t *testing.T) { + const encryptedKeyHex = "c18c032a67d68660df41c70104005789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8" + const expectedKeyHex = "d930363f7e0308c333b9618617ea728963d8df993665ae7be1092d4926fd864b" + + p, err := Read(readerFromHex(encryptedKeyHex)) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + ek, ok := p.(*EncryptedKey) + if !ok { + t.Errorf("didn't parse an EncryptedKey, got %#v", p) + return + } + + if ek.KeyId != 0x2a67d68660df41c7 || ek.Algo != PubKeyAlgoRSA { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + err = ek.Decrypt(encryptedKeyPriv, nil) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + if ek.CipherFunc != CipherAES256 { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + keyHex := fmt.Sprintf("%x", ek.Key) + if keyHex != expectedKeyHex { + t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex) + } +} + +func TestEncryptingEncryptedKey(t *testing.T) { + key := []byte{1, 2, 3, 4} + const expectedKeyHex = "01020304" + const keyId = 42 + + pub := &PublicKey{ + PublicKey: &encryptedKeyPub, + KeyId: keyId, + PubKeyAlgo: PubKeyAlgoRSAEncryptOnly, + } + + buf := new(bytes.Buffer) + err := SerializeEncryptedKey(buf, pub, CipherAES128, key, nil) + if err != nil { + t.Errorf("error writing encrypted key packet: %s", err) + } + + p, err := Read(buf) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + ek, ok := p.(*EncryptedKey) + if !ok { + t.Errorf("didn't parse an EncryptedKey, got %#v", p) + return + } + + if ek.KeyId != keyId || ek.Algo != PubKeyAlgoRSAEncryptOnly { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + err = ek.Decrypt(encryptedKeyPriv, nil) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + if ek.CipherFunc != CipherAES128 { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + keyHex := fmt.Sprintf("%x", ek.Key) + if keyHex != expectedKeyHex { + t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/literal.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/literal.go new file mode 100644 index 00000000000..1a9ec6e51e8 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/literal.go @@ -0,0 +1,89 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "encoding/binary" + "io" +) + +// LiteralData represents an encrypted file. See RFC 4880, section 5.9. +type LiteralData struct { + IsBinary bool + FileName string + Time uint32 // Unix epoch time. Either creation time or modification time. 0 means undefined. + Body io.Reader +} + +// ForEyesOnly returns whether the contents of the LiteralData have been marked +// as especially sensitive. +func (l *LiteralData) ForEyesOnly() bool { + return l.FileName == "_CONSOLE" +} + +func (l *LiteralData) parse(r io.Reader) (err error) { + var buf [256]byte + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + + l.IsBinary = buf[0] == 'b' + fileNameLen := int(buf[1]) + + _, err = readFull(r, buf[:fileNameLen]) + if err != nil { + return + } + + l.FileName = string(buf[:fileNameLen]) + + _, err = readFull(r, buf[:4]) + if err != nil { + return + } + + l.Time = binary.BigEndian.Uint32(buf[:4]) + l.Body = r + return +} + +// SerializeLiteral serializes a literal data packet to w and returns a +// WriteCloser to which the data itself can be written and which MUST be closed +// on completion. The fileName is truncated to 255 bytes. +func SerializeLiteral(w io.WriteCloser, isBinary bool, fileName string, time uint32) (plaintext io.WriteCloser, err error) { + var buf [4]byte + buf[0] = 't' + if isBinary { + buf[0] = 'b' + } + if len(fileName) > 255 { + fileName = fileName[:255] + } + buf[1] = byte(len(fileName)) + + inner, err := serializeStreamHeader(w, packetTypeLiteralData) + if err != nil { + return + } + + _, err = inner.Write(buf[:2]) + if err != nil { + return + } + _, err = inner.Write([]byte(fileName)) + if err != nil { + return + } + binary.BigEndian.PutUint32(buf[:], time) + _, err = inner.Write(buf[:]) + if err != nil { + return + } + + plaintext = inner + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/ocfb.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/ocfb.go new file mode 100644 index 00000000000..ce2a33a547c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/ocfb.go @@ -0,0 +1,143 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OpenPGP CFB Mode. http://tools.ietf.org/html/rfc4880#section-13.9 + +package packet + +import ( + "crypto/cipher" +) + +type ocfbEncrypter struct { + b cipher.Block + fre []byte + outUsed int +} + +// An OCFBResyncOption determines if the "resynchronization step" of OCFB is +// performed. +type OCFBResyncOption bool + +const ( + OCFBResync OCFBResyncOption = true + OCFBNoResync OCFBResyncOption = false +) + +// NewOCFBEncrypter returns a cipher.Stream which encrypts data with OpenPGP's +// cipher feedback mode using the given cipher.Block, and an initial amount of +// ciphertext. randData must be random bytes and be the same length as the +// cipher.Block's block size. Resync determines if the "resynchronization step" +// from RFC 4880, 13.9 step 7 is performed. Different parts of OpenPGP vary on +// this point. +func NewOCFBEncrypter(block cipher.Block, randData []byte, resync OCFBResyncOption) (cipher.Stream, []byte) { + blockSize := block.BlockSize() + if len(randData) != blockSize { + return nil, nil + } + + x := &ocfbEncrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefix := make([]byte, blockSize+2) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefix[i] = randData[i] ^ x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefix[blockSize] = x.fre[0] ^ randData[blockSize-2] + prefix[blockSize+1] = x.fre[1] ^ randData[blockSize-1] + + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } + return x, prefix +} + +func (x *ocfbEncrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + x.fre[x.outUsed] ^= src[i] + dst[i] = x.fre[x.outUsed] + x.outUsed++ + } +} + +type ocfbDecrypter struct { + b cipher.Block + fre []byte + outUsed int +} + +// NewOCFBDecrypter returns a cipher.Stream which decrypts data with OpenPGP's +// cipher feedback mode using the given cipher.Block. Prefix must be the first +// blockSize + 2 bytes of the ciphertext, where blockSize is the cipher.Block's +// block size. If an incorrect key is detected then nil is returned. On +// successful exit, blockSize+2 bytes of decrypted data are written into +// prefix. Resync determines if the "resynchronization step" from RFC 4880, +// 13.9 step 7 is performed. Different parts of OpenPGP vary on this point. +func NewOCFBDecrypter(block cipher.Block, prefix []byte, resync OCFBResyncOption) cipher.Stream { + blockSize := block.BlockSize() + if len(prefix) != blockSize+2 { + return nil + } + + x := &ocfbDecrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefixCopy := make([]byte, len(prefix)) + copy(prefixCopy, prefix) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefixCopy[i] ^= x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefixCopy[blockSize] ^= x.fre[0] + prefixCopy[blockSize+1] ^= x.fre[1] + + if prefixCopy[blockSize-2] != prefixCopy[blockSize] || + prefixCopy[blockSize-1] != prefixCopy[blockSize+1] { + return nil + } + + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } + copy(prefix, prefixCopy) + return x +} + +func (x *ocfbDecrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + c := src[i] + dst[i] = x.fre[x.outUsed] ^ src[i] + x.fre[x.outUsed] = c + x.outUsed++ + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/ocfb_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/ocfb_test.go new file mode 100644 index 00000000000..91022c042d4 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/ocfb_test.go @@ -0,0 +1,46 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/aes" + "crypto/rand" + "testing" +) + +var commonKey128 = []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c} + +func testOCFB(t *testing.T, resync OCFBResyncOption) { + block, err := aes.NewCipher(commonKey128) + if err != nil { + t.Error(err) + return + } + + plaintext := []byte("this is the plaintext, which is long enough to span several blocks.") + randData := make([]byte, block.BlockSize()) + rand.Reader.Read(randData) + ocfb, prefix := NewOCFBEncrypter(block, randData, resync) + ciphertext := make([]byte, len(plaintext)) + ocfb.XORKeyStream(ciphertext, plaintext) + + ocfbdec := NewOCFBDecrypter(block, prefix, resync) + if ocfbdec == nil { + t.Errorf("NewOCFBDecrypter failed (resync: %t)", resync) + return + } + plaintextCopy := make([]byte, len(plaintext)) + ocfbdec.XORKeyStream(plaintextCopy, ciphertext) + + if !bytes.Equal(plaintextCopy, plaintext) { + t.Errorf("got: %x, want: %x (resync: %t)", plaintextCopy, plaintext, resync) + } +} + +func TestOCFB(t *testing.T) { + testOCFB(t, OCFBNoResync) + testOCFB(t, OCFBResync) +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/one_pass_signature.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/one_pass_signature.go new file mode 100644 index 00000000000..1713503395e --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/one_pass_signature.go @@ -0,0 +1,73 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "encoding/binary" + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/s2k" + "io" + "strconv" +) + +// OnePassSignature represents a one-pass signature packet. See RFC 4880, +// section 5.4. +type OnePassSignature struct { + SigType SignatureType + Hash crypto.Hash + PubKeyAlgo PublicKeyAlgorithm + KeyId uint64 + IsLast bool +} + +const onePassSignatureVersion = 3 + +func (ops *OnePassSignature) parse(r io.Reader) (err error) { + var buf [13]byte + + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != onePassSignatureVersion { + err = errors.UnsupportedError("one-pass-signature packet version " + strconv.Itoa(int(buf[0]))) + } + + var ok bool + ops.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return errors.UnsupportedError("hash function: " + strconv.Itoa(int(buf[2]))) + } + + ops.SigType = SignatureType(buf[1]) + ops.PubKeyAlgo = PublicKeyAlgorithm(buf[3]) + ops.KeyId = binary.BigEndian.Uint64(buf[4:12]) + ops.IsLast = buf[12] != 0 + return +} + +// Serialize marshals the given OnePassSignature to w. +func (ops *OnePassSignature) Serialize(w io.Writer) error { + var buf [13]byte + buf[0] = onePassSignatureVersion + buf[1] = uint8(ops.SigType) + var ok bool + buf[2], ok = s2k.HashToHashId(ops.Hash) + if !ok { + return errors.UnsupportedError("hash type: " + strconv.Itoa(int(ops.Hash))) + } + buf[3] = uint8(ops.PubKeyAlgo) + binary.BigEndian.PutUint64(buf[4:12], ops.KeyId) + if ops.IsLast { + buf[12] = 1 + } + + if err := serializeHeader(w, packetTypeOnePassSignature, len(buf)); err != nil { + return err + } + _, err := w.Write(buf[:]) + return err +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/opaque.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/opaque.go new file mode 100644 index 00000000000..f2e00b98279 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/opaque.go @@ -0,0 +1,161 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "golang.org/x/crypto/openpgp/errors" + "io" + "io/ioutil" +) + +// OpaquePacket represents an OpenPGP packet as raw, unparsed data. This is +// useful for splitting and storing the original packet contents separately, +// handling unsupported packet types or accessing parts of the packet not yet +// implemented by this package. +type OpaquePacket struct { + // Packet type + Tag uint8 + // Reason why the packet was parsed opaquely + Reason error + // Binary contents of the packet data + Contents []byte +} + +func (op *OpaquePacket) parse(r io.Reader) (err error) { + op.Contents, err = ioutil.ReadAll(r) + return +} + +// Serialize marshals the packet to a writer in its original form, including +// the packet header. +func (op *OpaquePacket) Serialize(w io.Writer) (err error) { + err = serializeHeader(w, packetType(op.Tag), len(op.Contents)) + if err == nil { + _, err = w.Write(op.Contents) + } + return +} + +// Parse attempts to parse the opaque contents into a structure supported by +// this package. If the packet is not known then the result will be another +// OpaquePacket. +func (op *OpaquePacket) Parse() (p Packet, err error) { + hdr := bytes.NewBuffer(nil) + err = serializeHeader(hdr, packetType(op.Tag), len(op.Contents)) + if err != nil { + op.Reason = err + return op, err + } + p, err = Read(io.MultiReader(hdr, bytes.NewBuffer(op.Contents))) + if err != nil { + op.Reason = err + p = op + } + return +} + +// OpaqueReader reads OpaquePackets from an io.Reader. +type OpaqueReader struct { + r io.Reader +} + +func NewOpaqueReader(r io.Reader) *OpaqueReader { + return &OpaqueReader{r: r} +} + +// Read the next OpaquePacket. +func (or *OpaqueReader) Next() (op *OpaquePacket, err error) { + tag, _, contents, err := readHeader(or.r) + if err != nil { + return + } + op = &OpaquePacket{Tag: uint8(tag), Reason: err} + err = op.parse(contents) + if err != nil { + consumeAll(contents) + } + return +} + +// OpaqueSubpacket represents an unparsed OpenPGP subpacket, +// as found in signature and user attribute packets. +type OpaqueSubpacket struct { + SubType uint8 + Contents []byte +} + +// OpaqueSubpackets extracts opaque, unparsed OpenPGP subpackets from +// their byte representation. +func OpaqueSubpackets(contents []byte) (result []*OpaqueSubpacket, err error) { + var ( + subHeaderLen int + subPacket *OpaqueSubpacket + ) + for len(contents) > 0 { + subHeaderLen, subPacket, err = nextSubpacket(contents) + if err != nil { + break + } + result = append(result, subPacket) + contents = contents[subHeaderLen+len(subPacket.Contents):] + } + return +} + +func nextSubpacket(contents []byte) (subHeaderLen int, subPacket *OpaqueSubpacket, err error) { + // RFC 4880, section 5.2.3.1 + var subLen uint32 + if len(contents) < 1 { + goto Truncated + } + subPacket = &OpaqueSubpacket{} + switch { + case contents[0] < 192: + subHeaderLen = 2 // 1 length byte, 1 subtype byte + if len(contents) < subHeaderLen { + goto Truncated + } + subLen = uint32(contents[0]) + contents = contents[1:] + case contents[0] < 255: + subHeaderLen = 3 // 2 length bytes, 1 subtype + if len(contents) < subHeaderLen { + goto Truncated + } + subLen = uint32(contents[0]-192)<<8 + uint32(contents[1]) + 192 + contents = contents[2:] + default: + subHeaderLen = 6 // 5 length bytes, 1 subtype + if len(contents) < subHeaderLen { + goto Truncated + } + subLen = uint32(contents[1])<<24 | + uint32(contents[2])<<16 | + uint32(contents[3])<<8 | + uint32(contents[4]) + contents = contents[5:] + } + if subLen > uint32(len(contents)) { + goto Truncated + } + subPacket.SubType = contents[0] + subPacket.Contents = contents[1:subLen] + return +Truncated: + err = errors.StructuralError("subpacket truncated") + return +} + +func (osp *OpaqueSubpacket) Serialize(w io.Writer) (err error) { + buf := make([]byte, 6) + n := serializeSubpacketLength(buf, len(osp.Contents)+1) + buf[n] = osp.SubType + if _, err = w.Write(buf[:n+1]); err != nil { + return + } + _, err = w.Write(osp.Contents) + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/opaque_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/opaque_test.go new file mode 100644 index 00000000000..f27bbfe090b --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/opaque_test.go @@ -0,0 +1,67 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "io" + "testing" +) + +// Test packet.Read error handling in OpaquePacket.Parse, +// which attempts to re-read an OpaquePacket as a supported +// Packet type. +func TestOpaqueParseReason(t *testing.T) { + buf, err := hex.DecodeString(UnsupportedKeyHex) + if err != nil { + t.Fatal(err) + } + or := NewOpaqueReader(bytes.NewBuffer(buf)) + count := 0 + badPackets := 0 + var uid *UserId + for { + op, err := or.Next() + if err == io.EOF { + break + } else if err != nil { + t.Errorf("#%d: opaque read error: %v", count, err) + break + } + // try to parse opaque packet + p, err := op.Parse() + switch pkt := p.(type) { + case *UserId: + uid = pkt + case *OpaquePacket: + // If an OpaquePacket can't re-parse, packet.Read + // certainly had its reasons. + if pkt.Reason == nil { + t.Errorf("#%d: opaque packet, no reason", count) + } else { + badPackets++ + } + } + count++ + } + + const expectedBad = 3 + // Test post-conditions, make sure we actually parsed packets as expected. + if badPackets != expectedBad { + t.Errorf("unexpected # unparseable packets: %d (want %d)", badPackets, expectedBad) + } + if uid == nil { + t.Errorf("failed to find expected UID in unsupported keyring") + } else if uid.Id != "Armin M. Warda " { + t.Errorf("unexpected UID: %v", uid.Id) + } +} + +// This key material has public key and signature packet versions modified to +// an unsupported value (1), so that trying to parse the OpaquePacket to +// a typed packet will get an error. It also contains a GnuPG trust packet. +// (Created with: od -An -t x1 pubring.gpg | xargs | sed 's/ //g') +const UnsupportedKeyHex = `988d012e7a18a20000010400d6ac00d92b89c1f4396c243abb9b76d2e9673ad63483291fed88e22b82e255e441c078c6abbbf7d2d195e50b62eeaa915b85b0ec20c225ce2c64c167cacb6e711daf2e45da4a8356a059b8160e3b3628ac0dd8437b31f06d53d6e8ea4214d4a26406a6b63e1001406ef23e0bb3069fac9a99a91f77dfafd5de0f188a5da5e3c9000511b42741726d696e204d2e205761726461203c7761726461406e657068696c696d2e727568722e64653e8900950105102e8936c705d1eb399e58489901013f0e03ff5a0c4f421e34fcfa388129166420c08cd76987bcdec6f01bd0271459a85cc22048820dd4e44ac2c7d23908d540f54facf1b36b0d9c20488781ce9dca856531e76e2e846826e9951338020a03a09b57aa5faa82e9267458bd76105399885ac35af7dc1cbb6aaed7c39e1039f3b5beda2c0e916bd38560509bab81235d1a0ead83b0020000` diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/packet.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/packet.go new file mode 100644 index 00000000000..b1e7a7f0ba5 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/packet.go @@ -0,0 +1,538 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package packet implements parsing and serialization of OpenPGP packets, as +// specified in RFC 4880. +package packet + +import ( + "bufio" + "crypto/aes" + "crypto/cipher" + "crypto/des" + "golang.org/x/crypto/cast5" + "golang.org/x/crypto/openpgp/errors" + "io" + "math/big" +) + +// readFull is the same as io.ReadFull except that reading zero bytes returns +// ErrUnexpectedEOF rather than EOF. +func readFull(r io.Reader, buf []byte) (n int, err error) { + n, err = io.ReadFull(r, buf) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readLength reads an OpenPGP length from r. See RFC 4880, section 4.2.2. +func readLength(r io.Reader) (length int64, isPartial bool, err error) { + var buf [4]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + switch { + case buf[0] < 192: + length = int64(buf[0]) + case buf[0] < 224: + length = int64(buf[0]-192) << 8 + _, err = readFull(r, buf[0:1]) + if err != nil { + return + } + length += int64(buf[0]) + 192 + case buf[0] < 255: + length = int64(1) << (buf[0] & 0x1f) + isPartial = true + default: + _, err = readFull(r, buf[0:4]) + if err != nil { + return + } + length = int64(buf[0])<<24 | + int64(buf[1])<<16 | + int64(buf[2])<<8 | + int64(buf[3]) + } + return +} + +// partialLengthReader wraps an io.Reader and handles OpenPGP partial lengths. +// The continuation lengths are parsed and removed from the stream and EOF is +// returned at the end of the packet. See RFC 4880, section 4.2.2.4. +type partialLengthReader struct { + r io.Reader + remaining int64 + isPartial bool +} + +func (r *partialLengthReader) Read(p []byte) (n int, err error) { + for r.remaining == 0 { + if !r.isPartial { + return 0, io.EOF + } + r.remaining, r.isPartial, err = readLength(r.r) + if err != nil { + return 0, err + } + } + + toRead := int64(len(p)) + if toRead > r.remaining { + toRead = r.remaining + } + + n, err = r.r.Read(p[:int(toRead)]) + r.remaining -= int64(n) + if n < int(toRead) && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// partialLengthWriter writes a stream of data using OpenPGP partial lengths. +// See RFC 4880, section 4.2.2.4. +type partialLengthWriter struct { + w io.WriteCloser + lengthByte [1]byte +} + +func (w *partialLengthWriter) Write(p []byte) (n int, err error) { + for len(p) > 0 { + for power := uint(14); power < 32; power-- { + l := 1 << power + if len(p) >= l { + w.lengthByte[0] = 224 + uint8(power) + _, err = w.w.Write(w.lengthByte[:]) + if err != nil { + return + } + var m int + m, err = w.w.Write(p[:l]) + n += m + if err != nil { + return + } + p = p[l:] + break + } + } + } + return +} + +func (w *partialLengthWriter) Close() error { + w.lengthByte[0] = 0 + _, err := w.w.Write(w.lengthByte[:]) + if err != nil { + return err + } + return w.w.Close() +} + +// A spanReader is an io.LimitReader, but it returns ErrUnexpectedEOF if the +// underlying Reader returns EOF before the limit has been reached. +type spanReader struct { + r io.Reader + n int64 +} + +func (l *spanReader) Read(p []byte) (n int, err error) { + if l.n <= 0 { + return 0, io.EOF + } + if int64(len(p)) > l.n { + p = p[0:l.n] + } + n, err = l.r.Read(p) + l.n -= int64(n) + if l.n > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readHeader parses a packet header and returns an io.Reader which will return +// the contents of the packet. See RFC 4880, section 4.2. +func readHeader(r io.Reader) (tag packetType, length int64, contents io.Reader, err error) { + var buf [4]byte + _, err = io.ReadFull(r, buf[:1]) + if err != nil { + return + } + if buf[0]&0x80 == 0 { + err = errors.StructuralError("tag byte does not have MSB set") + return + } + if buf[0]&0x40 == 0 { + // Old format packet + tag = packetType((buf[0] & 0x3f) >> 2) + lengthType := buf[0] & 3 + if lengthType == 3 { + length = -1 + contents = r + return + } + lengthBytes := 1 << lengthType + _, err = readFull(r, buf[0:lengthBytes]) + if err != nil { + return + } + for i := 0; i < lengthBytes; i++ { + length <<= 8 + length |= int64(buf[i]) + } + contents = &spanReader{r, length} + return + } + + // New format packet + tag = packetType(buf[0] & 0x3f) + length, isPartial, err := readLength(r) + if err != nil { + return + } + if isPartial { + contents = &partialLengthReader{ + remaining: length, + isPartial: true, + r: r, + } + length = -1 + } else { + contents = &spanReader{r, length} + } + return +} + +// serializeHeader writes an OpenPGP packet header to w. See RFC 4880, section +// 4.2. +func serializeHeader(w io.Writer, ptype packetType, length int) (err error) { + var buf [6]byte + var n int + + buf[0] = 0x80 | 0x40 | byte(ptype) + if length < 192 { + buf[1] = byte(length) + n = 2 + } else if length < 8384 { + length -= 192 + buf[1] = 192 + byte(length>>8) + buf[2] = byte(length) + n = 3 + } else { + buf[1] = 255 + buf[2] = byte(length >> 24) + buf[3] = byte(length >> 16) + buf[4] = byte(length >> 8) + buf[5] = byte(length) + n = 6 + } + + _, err = w.Write(buf[:n]) + return +} + +// serializeStreamHeader writes an OpenPGP packet header to w where the +// length of the packet is unknown. It returns a io.WriteCloser which can be +// used to write the contents of the packet. See RFC 4880, section 4.2. +func serializeStreamHeader(w io.WriteCloser, ptype packetType) (out io.WriteCloser, err error) { + var buf [1]byte + buf[0] = 0x80 | 0x40 | byte(ptype) + _, err = w.Write(buf[:]) + if err != nil { + return + } + out = &partialLengthWriter{w: w} + return +} + +// Packet represents an OpenPGP packet. Users are expected to try casting +// instances of this interface to specific packet types. +type Packet interface { + parse(io.Reader) error +} + +// consumeAll reads from the given Reader until error, returning the number of +// bytes read. +func consumeAll(r io.Reader) (n int64, err error) { + var m int + var buf [1024]byte + + for { + m, err = r.Read(buf[:]) + n += int64(m) + if err == io.EOF { + err = nil + return + } + if err != nil { + return + } + } + + panic("unreachable") +} + +// packetType represents the numeric ids of the different OpenPGP packet types. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-2 +type packetType uint8 + +const ( + packetTypeEncryptedKey packetType = 1 + packetTypeSignature packetType = 2 + packetTypeSymmetricKeyEncrypted packetType = 3 + packetTypeOnePassSignature packetType = 4 + packetTypePrivateKey packetType = 5 + packetTypePublicKey packetType = 6 + packetTypePrivateSubkey packetType = 7 + packetTypeCompressed packetType = 8 + packetTypeSymmetricallyEncrypted packetType = 9 + packetTypeLiteralData packetType = 11 + packetTypeUserId packetType = 13 + packetTypePublicSubkey packetType = 14 + packetTypeUserAttribute packetType = 17 + packetTypeSymmetricallyEncryptedMDC packetType = 18 +) + +// peekVersion detects the version of a public key packet about to +// be read. A bufio.Reader at the original position of the io.Reader +// is returned. +func peekVersion(r io.Reader) (bufr *bufio.Reader, ver byte, err error) { + bufr = bufio.NewReader(r) + var verBuf []byte + if verBuf, err = bufr.Peek(1); err != nil { + return + } + ver = verBuf[0] + return +} + +// Read reads a single OpenPGP packet from the given io.Reader. If there is an +// error parsing a packet, the whole packet is consumed from the input. +func Read(r io.Reader) (p Packet, err error) { + tag, _, contents, err := readHeader(r) + if err != nil { + return + } + + switch tag { + case packetTypeEncryptedKey: + p = new(EncryptedKey) + case packetTypeSignature: + var version byte + // Detect signature version + if contents, version, err = peekVersion(contents); err != nil { + return + } + if version < 4 { + p = new(SignatureV3) + } else { + p = new(Signature) + } + case packetTypeSymmetricKeyEncrypted: + p = new(SymmetricKeyEncrypted) + case packetTypeOnePassSignature: + p = new(OnePassSignature) + case packetTypePrivateKey, packetTypePrivateSubkey: + pk := new(PrivateKey) + if tag == packetTypePrivateSubkey { + pk.IsSubkey = true + } + p = pk + case packetTypePublicKey, packetTypePublicSubkey: + var version byte + if contents, version, err = peekVersion(contents); err != nil { + return + } + isSubkey := tag == packetTypePublicSubkey + if version < 4 { + p = &PublicKeyV3{IsSubkey: isSubkey} + } else { + p = &PublicKey{IsSubkey: isSubkey} + } + case packetTypeCompressed: + p = new(Compressed) + case packetTypeSymmetricallyEncrypted: + p = new(SymmetricallyEncrypted) + case packetTypeLiteralData: + p = new(LiteralData) + case packetTypeUserId: + p = new(UserId) + case packetTypeUserAttribute: + p = new(UserAttribute) + case packetTypeSymmetricallyEncryptedMDC: + se := new(SymmetricallyEncrypted) + se.MDC = true + p = se + default: + err = errors.UnknownPacketTypeError(tag) + } + if p != nil { + err = p.parse(contents) + } + if err != nil { + consumeAll(contents) + } + return +} + +// SignatureType represents the different semantic meanings of an OpenPGP +// signature. See RFC 4880, section 5.2.1. +type SignatureType uint8 + +const ( + SigTypeBinary SignatureType = 0 + SigTypeText = 1 + SigTypeGenericCert = 0x10 + SigTypePersonaCert = 0x11 + SigTypeCasualCert = 0x12 + SigTypePositiveCert = 0x13 + SigTypeSubkeyBinding = 0x18 + SigTypeDirectSignature = 0x1F + SigTypeKeyRevocation = 0x20 + SigTypeSubkeyRevocation = 0x28 +) + +// PublicKeyAlgorithm represents the different public key system specified for +// OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-12 +type PublicKeyAlgorithm uint8 + +const ( + PubKeyAlgoRSA PublicKeyAlgorithm = 1 + PubKeyAlgoRSAEncryptOnly PublicKeyAlgorithm = 2 + PubKeyAlgoRSASignOnly PublicKeyAlgorithm = 3 + PubKeyAlgoElGamal PublicKeyAlgorithm = 16 + PubKeyAlgoDSA PublicKeyAlgorithm = 17 + // RFC 6637, Section 5. + PubKeyAlgoECDH PublicKeyAlgorithm = 18 + PubKeyAlgoECDSA PublicKeyAlgorithm = 19 +) + +// CanEncrypt returns true if it's possible to encrypt a message to a public +// key of the given type. +func (pka PublicKeyAlgorithm) CanEncrypt() bool { + switch pka { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoElGamal: + return true + } + return false +} + +// CanSign returns true if it's possible for a public key of the given type to +// sign a message. +func (pka PublicKeyAlgorithm) CanSign() bool { + switch pka { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoDSA: + return true + } + return false +} + +// CipherFunction represents the different block ciphers specified for OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-13 +type CipherFunction uint8 + +const ( + Cipher3DES CipherFunction = 2 + CipherCAST5 CipherFunction = 3 + CipherAES128 CipherFunction = 7 + CipherAES192 CipherFunction = 8 + CipherAES256 CipherFunction = 9 +) + +// KeySize returns the key size, in bytes, of cipher. +func (cipher CipherFunction) KeySize() int { + switch cipher { + case Cipher3DES: + return 24 + case CipherCAST5: + return cast5.KeySize + case CipherAES128: + return 16 + case CipherAES192: + return 24 + case CipherAES256: + return 32 + } + return 0 +} + +// blockSize returns the block size, in bytes, of cipher. +func (cipher CipherFunction) blockSize() int { + switch cipher { + case Cipher3DES: + return des.BlockSize + case CipherCAST5: + return 8 + case CipherAES128, CipherAES192, CipherAES256: + return 16 + } + return 0 +} + +// new returns a fresh instance of the given cipher. +func (cipher CipherFunction) new(key []byte) (block cipher.Block) { + switch cipher { + case Cipher3DES: + block, _ = des.NewTripleDESCipher(key) + case CipherCAST5: + block, _ = cast5.NewCipher(key) + case CipherAES128, CipherAES192, CipherAES256: + block, _ = aes.NewCipher(key) + } + return +} + +// readMPI reads a big integer from r. The bit length returned is the bit +// length that was specified in r. This is preserved so that the integer can be +// reserialized exactly. +func readMPI(r io.Reader) (mpi []byte, bitLength uint16, err error) { + var buf [2]byte + _, err = readFull(r, buf[0:]) + if err != nil { + return + } + bitLength = uint16(buf[0])<<8 | uint16(buf[1]) + numBytes := (int(bitLength) + 7) / 8 + mpi = make([]byte, numBytes) + _, err = readFull(r, mpi) + return +} + +// mpiLength returns the length of the given *big.Int when serialized as an +// MPI. +func mpiLength(n *big.Int) (mpiLengthInBytes int) { + mpiLengthInBytes = 2 /* MPI length */ + mpiLengthInBytes += (n.BitLen() + 7) / 8 + return +} + +// writeMPI serializes a big integer to w. +func writeMPI(w io.Writer, bitLength uint16, mpiBytes []byte) (err error) { + _, err = w.Write([]byte{byte(bitLength >> 8), byte(bitLength)}) + if err == nil { + _, err = w.Write(mpiBytes) + } + return +} + +// writeBig serializes a *big.Int to w. +func writeBig(w io.Writer, i *big.Int) error { + return writeMPI(w, uint16(i.BitLen()), i.Bytes()) +} + +// CompressionAlgo Represents the different compression algorithms +// supported by OpenPGP (except for BZIP2, which is not currently +// supported). See Section 9.3 of RFC 4880. +type CompressionAlgo uint8 + +const ( + CompressionNone CompressionAlgo = 0 + CompressionZIP CompressionAlgo = 1 + CompressionZLIB CompressionAlgo = 2 +) diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/packet_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/packet_test.go new file mode 100644 index 00000000000..1dab5c3d588 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/packet_test.go @@ -0,0 +1,255 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "fmt" + "golang.org/x/crypto/openpgp/errors" + "io" + "io/ioutil" + "testing" +) + +func TestReadFull(t *testing.T) { + var out [4]byte + + b := bytes.NewBufferString("foo") + n, err := readFull(b, out[:3]) + if n != 3 || err != nil { + t.Errorf("full read failed n:%d err:%s", n, err) + } + + b = bytes.NewBufferString("foo") + n, err = readFull(b, out[:4]) + if n != 3 || err != io.ErrUnexpectedEOF { + t.Errorf("partial read failed n:%d err:%s", n, err) + } + + b = bytes.NewBuffer(nil) + n, err = readFull(b, out[:3]) + if n != 0 || err != io.ErrUnexpectedEOF { + t.Errorf("empty read failed n:%d err:%s", n, err) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +var readLengthTests = []struct { + hexInput string + length int64 + isPartial bool + err error +}{ + {"", 0, false, io.ErrUnexpectedEOF}, + {"1f", 31, false, nil}, + {"c0", 0, false, io.ErrUnexpectedEOF}, + {"c101", 256 + 1 + 192, false, nil}, + {"e0", 1, true, nil}, + {"e1", 2, true, nil}, + {"e2", 4, true, nil}, + {"ff", 0, false, io.ErrUnexpectedEOF}, + {"ff00", 0, false, io.ErrUnexpectedEOF}, + {"ff0000", 0, false, io.ErrUnexpectedEOF}, + {"ff000000", 0, false, io.ErrUnexpectedEOF}, + {"ff00000000", 0, false, nil}, + {"ff01020304", 16909060, false, nil}, +} + +func TestReadLength(t *testing.T) { + for i, test := range readLengthTests { + length, isPartial, err := readLength(readerFromHex(test.hexInput)) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + if length != test.length || isPartial != test.isPartial { + t.Errorf("%d: bad result got:(%d,%t) want:(%d,%t)", i, length, isPartial, test.length, test.isPartial) + } + } +} + +var partialLengthReaderTests = []struct { + hexInput string + err error + hexOutput string +}{ + {"e0", io.ErrUnexpectedEOF, ""}, + {"e001", io.ErrUnexpectedEOF, ""}, + {"e0010102", nil, "0102"}, + {"ff00000000", nil, ""}, + {"e10102e1030400", nil, "01020304"}, + {"e101", io.ErrUnexpectedEOF, ""}, +} + +func TestPartialLengthReader(t *testing.T) { + for i, test := range partialLengthReaderTests { + r := &partialLengthReader{readerFromHex(test.hexInput), 0, true} + out, err := ioutil.ReadAll(r) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + + got := fmt.Sprintf("%x", out) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, test.hexOutput, got) + } + } +} + +var readHeaderTests = []struct { + hexInput string + structuralError bool + unexpectedEOF bool + tag int + length int64 + hexOutput string +}{ + {"", false, false, 0, 0, ""}, + {"7f", true, false, 0, 0, ""}, + + // Old format headers + {"80", false, true, 0, 0, ""}, + {"8001", false, true, 0, 1, ""}, + {"800102", false, false, 0, 1, "02"}, + {"81000102", false, false, 0, 1, "02"}, + {"820000000102", false, false, 0, 1, "02"}, + {"860000000102", false, false, 1, 1, "02"}, + {"83010203", false, false, 0, -1, "010203"}, + + // New format headers + {"c0", false, true, 0, 0, ""}, + {"c000", false, false, 0, 0, ""}, + {"c00102", false, false, 0, 1, "02"}, + {"c0020203", false, false, 0, 2, "0203"}, + {"c00202", false, true, 0, 2, ""}, + {"c3020203", false, false, 3, 2, "0203"}, +} + +func TestReadHeader(t *testing.T) { + for i, test := range readHeaderTests { + tag, length, contents, err := readHeader(readerFromHex(test.hexInput)) + if test.structuralError { + if _, ok := err.(errors.StructuralError); ok { + continue + } + t.Errorf("%d: expected StructuralError, got:%s", i, err) + continue + } + if err != nil { + if len(test.hexInput) == 0 && err == io.EOF { + continue + } + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from readHeader: %s", i, err) + } + continue + } + if int(tag) != test.tag || length != test.length { + t.Errorf("%d: got:(%d,%d) want:(%d,%d)", i, int(tag), length, test.tag, test.length) + continue + } + + body, err := ioutil.ReadAll(contents) + if err != nil { + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from contents: %s", i, err) + } + continue + } + if test.unexpectedEOF { + t.Errorf("%d: expected ErrUnexpectedEOF from contents but got no error", i) + continue + } + got := fmt.Sprintf("%x", body) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, got, test.hexOutput) + } + } +} + +func TestSerializeHeader(t *testing.T) { + tag := packetTypePublicKey + lengths := []int{0, 1, 2, 64, 192, 193, 8000, 8384, 8385, 10000} + + for _, length := range lengths { + buf := bytes.NewBuffer(nil) + serializeHeader(buf, tag, length) + tag2, length2, _, err := readHeader(buf) + if err != nil { + t.Errorf("length %d, err: %s", length, err) + } + if tag2 != tag { + t.Errorf("length %d, tag incorrect (got %d, want %d)", length, tag2, tag) + } + if int(length2) != length { + t.Errorf("length %d, length incorrect (got %d)", length, length2) + } + } +} + +func TestPartialLengths(t *testing.T) { + buf := bytes.NewBuffer(nil) + w := new(partialLengthWriter) + w.w = noOpCloser{buf} + + const maxChunkSize = 64 + + var b [maxChunkSize]byte + var n uint8 + for l := 1; l <= maxChunkSize; l++ { + for i := 0; i < l; i++ { + b[i] = n + n++ + } + m, err := w.Write(b[:l]) + if m != l { + t.Errorf("short write got: %d want: %d", m, l) + } + if err != nil { + t.Errorf("error from write: %s", err) + } + } + w.Close() + + want := (maxChunkSize * (maxChunkSize + 1)) / 2 + copyBuf := bytes.NewBuffer(nil) + r := &partialLengthReader{buf, 0, true} + m, err := io.Copy(copyBuf, r) + if m != int64(want) { + t.Errorf("short copy got: %d want: %d", m, want) + } + if err != nil { + t.Errorf("error from copy: %s", err) + } + + copyBytes := copyBuf.Bytes() + for i := 0; i < want; i++ { + if copyBytes[i] != uint8(i) { + t.Errorf("bad pattern in copy at %d", i) + break + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/private_key.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/private_key.go new file mode 100644 index 00000000000..9685a342c41 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/private_key.go @@ -0,0 +1,310 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/cipher" + "crypto/dsa" + "crypto/rsa" + "crypto/sha1" + "golang.org/x/crypto/openpgp/elgamal" + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/s2k" + "io" + "io/ioutil" + "math/big" + "strconv" + "time" +) + +// PrivateKey represents a possibly encrypted private key. See RFC 4880, +// section 5.5.3. +type PrivateKey struct { + PublicKey + Encrypted bool // if true then the private key is unavailable until Decrypt has been called. + encryptedData []byte + cipher CipherFunction + s2k func(out, in []byte) + PrivateKey interface{} // An *rsa.PrivateKey or *dsa.PrivateKey. + sha1Checksum bool + iv []byte +} + +func NewRSAPrivateKey(currentTime time.Time, priv *rsa.PrivateKey) *PrivateKey { + pk := new(PrivateKey) + pk.PublicKey = *NewRSAPublicKey(currentTime, &priv.PublicKey) + pk.PrivateKey = priv + return pk +} + +func NewDSAPrivateKey(currentTime time.Time, priv *dsa.PrivateKey) *PrivateKey { + pk := new(PrivateKey) + pk.PublicKey = *NewDSAPublicKey(currentTime, &priv.PublicKey) + pk.PrivateKey = priv + return pk +} + +func (pk *PrivateKey) parse(r io.Reader) (err error) { + err = (&pk.PublicKey).parse(r) + if err != nil { + return + } + var buf [1]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + + s2kType := buf[0] + + switch s2kType { + case 0: + pk.s2k = nil + pk.Encrypted = false + case 254, 255: + _, err = readFull(r, buf[:]) + if err != nil { + return + } + pk.cipher = CipherFunction(buf[0]) + pk.Encrypted = true + pk.s2k, err = s2k.Parse(r) + if err != nil { + return + } + if s2kType == 254 { + pk.sha1Checksum = true + } + default: + return errors.UnsupportedError("deprecated s2k function in private key") + } + + if pk.Encrypted { + blockSize := pk.cipher.blockSize() + if blockSize == 0 { + return errors.UnsupportedError("unsupported cipher in private key: " + strconv.Itoa(int(pk.cipher))) + } + pk.iv = make([]byte, blockSize) + _, err = readFull(r, pk.iv) + if err != nil { + return + } + } + + pk.encryptedData, err = ioutil.ReadAll(r) + if err != nil { + return + } + + if !pk.Encrypted { + return pk.parsePrivateKey(pk.encryptedData) + } + + return +} + +func mod64kHash(d []byte) uint16 { + var h uint16 + for _, b := range d { + h += uint16(b) + } + return h +} + +func (pk *PrivateKey) Serialize(w io.Writer) (err error) { + // TODO(agl): support encrypted private keys + buf := bytes.NewBuffer(nil) + err = pk.PublicKey.serializeWithoutHeaders(buf) + if err != nil { + return + } + buf.WriteByte(0 /* no encryption */) + + privateKeyBuf := bytes.NewBuffer(nil) + + switch priv := pk.PrivateKey.(type) { + case *rsa.PrivateKey: + err = serializeRSAPrivateKey(privateKeyBuf, priv) + case *dsa.PrivateKey: + err = serializeDSAPrivateKey(privateKeyBuf, priv) + default: + err = errors.InvalidArgumentError("unknown private key type") + } + if err != nil { + return + } + + ptype := packetTypePrivateKey + contents := buf.Bytes() + privateKeyBytes := privateKeyBuf.Bytes() + if pk.IsSubkey { + ptype = packetTypePrivateSubkey + } + err = serializeHeader(w, ptype, len(contents)+len(privateKeyBytes)+2) + if err != nil { + return + } + _, err = w.Write(contents) + if err != nil { + return + } + _, err = w.Write(privateKeyBytes) + if err != nil { + return + } + + checksum := mod64kHash(privateKeyBytes) + var checksumBytes [2]byte + checksumBytes[0] = byte(checksum >> 8) + checksumBytes[1] = byte(checksum) + _, err = w.Write(checksumBytes[:]) + + return +} + +func serializeRSAPrivateKey(w io.Writer, priv *rsa.PrivateKey) error { + err := writeBig(w, priv.D) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[1]) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[0]) + if err != nil { + return err + } + return writeBig(w, priv.Precomputed.Qinv) +} + +func serializeDSAPrivateKey(w io.Writer, priv *dsa.PrivateKey) error { + return writeBig(w, priv.X) +} + +// Decrypt decrypts an encrypted private key using a passphrase. +func (pk *PrivateKey) Decrypt(passphrase []byte) error { + if !pk.Encrypted { + return nil + } + + key := make([]byte, pk.cipher.KeySize()) + pk.s2k(key, passphrase) + block := pk.cipher.new(key) + cfb := cipher.NewCFBDecrypter(block, pk.iv) + + data := make([]byte, len(pk.encryptedData)) + cfb.XORKeyStream(data, pk.encryptedData) + + if pk.sha1Checksum { + if len(data) < sha1.Size { + return errors.StructuralError("truncated private key data") + } + h := sha1.New() + h.Write(data[:len(data)-sha1.Size]) + sum := h.Sum(nil) + if !bytes.Equal(sum, data[len(data)-sha1.Size:]) { + return errors.StructuralError("private key checksum failure") + } + data = data[:len(data)-sha1.Size] + } else { + if len(data) < 2 { + return errors.StructuralError("truncated private key data") + } + var sum uint16 + for i := 0; i < len(data)-2; i++ { + sum += uint16(data[i]) + } + if data[len(data)-2] != uint8(sum>>8) || + data[len(data)-1] != uint8(sum) { + return errors.StructuralError("private key checksum failure") + } + data = data[:len(data)-2] + } + + return pk.parsePrivateKey(data) +} + +func (pk *PrivateKey) parsePrivateKey(data []byte) (err error) { + switch pk.PublicKey.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoRSAEncryptOnly: + return pk.parseRSAPrivateKey(data) + case PubKeyAlgoDSA: + return pk.parseDSAPrivateKey(data) + case PubKeyAlgoElGamal: + return pk.parseElGamalPrivateKey(data) + } + panic("impossible") +} + +func (pk *PrivateKey) parseRSAPrivateKey(data []byte) (err error) { + rsaPub := pk.PublicKey.PublicKey.(*rsa.PublicKey) + rsaPriv := new(rsa.PrivateKey) + rsaPriv.PublicKey = *rsaPub + + buf := bytes.NewBuffer(data) + d, _, err := readMPI(buf) + if err != nil { + return + } + p, _, err := readMPI(buf) + if err != nil { + return + } + q, _, err := readMPI(buf) + if err != nil { + return + } + + rsaPriv.D = new(big.Int).SetBytes(d) + rsaPriv.Primes = make([]*big.Int, 2) + rsaPriv.Primes[0] = new(big.Int).SetBytes(p) + rsaPriv.Primes[1] = new(big.Int).SetBytes(q) + rsaPriv.Precompute() + pk.PrivateKey = rsaPriv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} + +func (pk *PrivateKey) parseDSAPrivateKey(data []byte) (err error) { + dsaPub := pk.PublicKey.PublicKey.(*dsa.PublicKey) + dsaPriv := new(dsa.PrivateKey) + dsaPriv.PublicKey = *dsaPub + + buf := bytes.NewBuffer(data) + x, _, err := readMPI(buf) + if err != nil { + return + } + + dsaPriv.X = new(big.Int).SetBytes(x) + pk.PrivateKey = dsaPriv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} + +func (pk *PrivateKey) parseElGamalPrivateKey(data []byte) (err error) { + pub := pk.PublicKey.PublicKey.(*elgamal.PublicKey) + priv := new(elgamal.PrivateKey) + priv.PublicKey = *pub + + buf := bytes.NewBuffer(data) + x, _, err := readMPI(buf) + if err != nil { + return + } + + priv.X = new(big.Int).SetBytes(x) + pk.PrivateKey = priv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/private_key_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/private_key_test.go new file mode 100644 index 00000000000..6a6197aec41 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/private_key_test.go @@ -0,0 +1,64 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" + "time" +) + +var privateKeyTests = []struct { + privateKeyHex string + creationTime time.Time +}{ + { + privKeyRSAHex, + time.Unix(0x4cc349a8, 0), + }, + { + privKeyElGamalHex, + time.Unix(0x4df9ee1a, 0), + }, +} + +func TestPrivateKeyRead(t *testing.T) { + for i, test := range privateKeyTests { + packet, err := Read(readerFromHex(test.privateKeyHex)) + if err != nil { + t.Errorf("#%d: failed to parse: %s", i, err) + continue + } + + privKey := packet.(*PrivateKey) + + if !privKey.Encrypted { + t.Errorf("#%d: private key isn't encrypted", i) + continue + } + + err = privKey.Decrypt([]byte("wrong password")) + if err == nil { + t.Errorf("#%d: decrypted with incorrect key", i) + continue + } + + err = privKey.Decrypt([]byte("testing")) + if err != nil { + t.Errorf("#%d: failed to decrypt: %s", i, err) + continue + } + + if !privKey.CreationTime.Equal(test.creationTime) || privKey.Encrypted { + t.Errorf("#%d: bad result, got: %#v", i, privKey) + } + } +} + +// Generated with `gpg --export-secret-keys "Test Key 2"` +const privKeyRSAHex = "9501fe044cc349a8010400b70ca0010e98c090008d45d1ee8f9113bd5861fd57b88bacb7c68658747663f1e1a3b5a98f32fda6472373c024b97359cd2efc88ff60f77751adfbf6af5e615e6a1408cfad8bf0cea30b0d5f53aa27ad59089ba9b15b7ebc2777a25d7b436144027e3bcd203909f147d0e332b240cf63d3395f5dfe0df0a6c04e8655af7eacdf0011010001fe0303024a252e7d475fd445607de39a265472aa74a9320ba2dac395faa687e9e0336aeb7e9a7397e511b5afd9dc84557c80ac0f3d4d7bfec5ae16f20d41c8c84a04552a33870b930420e230e179564f6d19bb153145e76c33ae993886c388832b0fa042ddda7f133924f3854481533e0ede31d51278c0519b29abc3bf53da673e13e3e1214b52413d179d7f66deee35cac8eacb060f78379d70ef4af8607e68131ff529439668fc39c9ce6dfef8a5ac234d234802cbfb749a26107db26406213ae5c06d4673253a3cbee1fcbae58d6ab77e38d6e2c0e7c6317c48e054edadb5a40d0d48acb44643d998139a8a66bb820be1f3f80185bc777d14b5954b60effe2448a036d565c6bc0b915fcea518acdd20ab07bc1529f561c58cd044f723109b93f6fd99f876ff891d64306b5d08f48bab59f38695e9109c4dec34013ba3153488ce070268381ba923ee1eb77125b36afcb4347ec3478c8f2735b06ef17351d872e577fa95d0c397c88c71b59629a36aec" + +// Generated by `gpg --export-secret-keys` followed by a manual extraction of +// the ElGamal subkey from the packets. +const privKeyElGamalHex = "9d0157044df9ee1a100400eb8e136a58ec39b582629cdadf830bc64e0a94ed8103ca8bb247b27b11b46d1d25297ef4bcc3071785ba0c0bedfe89eabc5287fcc0edf81ab5896c1c8e4b20d27d79813c7aede75320b33eaeeaa586edc00fd1036c10133e6ba0ff277245d0d59d04b2b3421b7244aca5f4a8d870c6f1c1fbff9e1c26699a860b9504f35ca1d700030503fd1ededd3b840795be6d9ccbe3c51ee42e2f39233c432b831ddd9c4e72b7025a819317e47bf94f9ee316d7273b05d5fcf2999c3a681f519b1234bbfa6d359b4752bd9c3f77d6b6456cde152464763414ca130f4e91d91041432f90620fec0e6d6b5116076c2985d5aeaae13be492b9b329efcaf7ee25120159a0a30cd976b42d7afe030302dae7eb80db744d4960c4df930d57e87fe81412eaace9f900e6c839817a614ddb75ba6603b9417c33ea7b6c93967dfa2bcff3fa3c74a5ce2c962db65b03aece14c96cbd0038fc" diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key.go new file mode 100644 index 00000000000..26bd9ccdd22 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key.go @@ -0,0 +1,687 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto" + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "crypto/sha1" + _ "crypto/sha256" + _ "crypto/sha512" + "encoding/binary" + "fmt" + "golang.org/x/crypto/openpgp/elgamal" + "golang.org/x/crypto/openpgp/errors" + "hash" + "io" + "math/big" + "strconv" + "time" +) + +var ( + // NIST curve P-256 + oidCurveP256 []byte = []byte{0x2A, 0x86, 0x48, 0xCE, 0x3D, 0x03, 0x01, 0x07} + // NIST curve P-384 + oidCurveP384 []byte = []byte{0x2B, 0x81, 0x04, 0x00, 0x22} + // NIST curve P-521 + oidCurveP521 []byte = []byte{0x2B, 0x81, 0x04, 0x00, 0x23} +) + +const maxOIDLength = 8 + +// ecdsaKey stores the algorithm-specific fields for ECDSA keys. +// as defined in RFC 6637, Section 9. +type ecdsaKey struct { + // oid contains the OID byte sequence identifying the elliptic curve used + oid []byte + // p contains the elliptic curve point that represents the public key + p parsedMPI +} + +// parseOID reads the OID for the curve as defined in RFC 6637, Section 9. +func parseOID(r io.Reader) (oid []byte, err error) { + buf := make([]byte, maxOIDLength) + if _, err = readFull(r, buf[:1]); err != nil { + return + } + oidLen := buf[0] + if int(oidLen) > len(buf) { + err = errors.UnsupportedError("invalid oid length: " + strconv.Itoa(int(oidLen))) + return + } + oid = buf[:oidLen] + _, err = readFull(r, oid) + return +} + +func (f *ecdsaKey) parse(r io.Reader) (err error) { + if f.oid, err = parseOID(r); err != nil { + return err + } + f.p.bytes, f.p.bitLength, err = readMPI(r) + return +} + +func (f *ecdsaKey) serialize(w io.Writer) (err error) { + buf := make([]byte, maxOIDLength+1) + buf[0] = byte(len(f.oid)) + copy(buf[1:], f.oid) + if _, err = w.Write(buf[:len(f.oid)+1]); err != nil { + return + } + return writeMPIs(w, f.p) +} + +func (f *ecdsaKey) newECDSA() (*ecdsa.PublicKey, error) { + var c elliptic.Curve + if bytes.Equal(f.oid, oidCurveP256) { + c = elliptic.P256() + } else if bytes.Equal(f.oid, oidCurveP384) { + c = elliptic.P384() + } else if bytes.Equal(f.oid, oidCurveP521) { + c = elliptic.P521() + } else { + return nil, errors.UnsupportedError(fmt.Sprintf("unsupported oid: %x", f.oid)) + } + x, y := elliptic.Unmarshal(c, f.p.bytes) + if x == nil { + return nil, errors.UnsupportedError("failed to parse EC point") + } + return &ecdsa.PublicKey{Curve: c, X: x, Y: y}, nil +} + +func (f *ecdsaKey) byteLen() int { + return 1 + len(f.oid) + 2 + len(f.p.bytes) +} + +type kdfHashFunction byte +type kdfAlgorithm byte + +// ecdhKdf stores key derivation function parameters +// used for ECDH encryption. See RFC 6637, Section 9. +type ecdhKdf struct { + KdfHash kdfHashFunction + KdfAlgo kdfAlgorithm +} + +func (f *ecdhKdf) parse(r io.Reader) (err error) { + buf := make([]byte, 1) + if _, err = readFull(r, buf); err != nil { + return + } + kdfLen := int(buf[0]) + if kdfLen < 3 { + return errors.UnsupportedError("Unsupported ECDH KDF length: " + strconv.Itoa(kdfLen)) + } + buf = make([]byte, kdfLen) + if _, err = readFull(r, buf); err != nil { + return + } + reserved := int(buf[0]) + f.KdfHash = kdfHashFunction(buf[1]) + f.KdfAlgo = kdfAlgorithm(buf[2]) + if reserved != 0x01 { + return errors.UnsupportedError("Unsupported KDF reserved field: " + strconv.Itoa(reserved)) + } + return +} + +func (f *ecdhKdf) serialize(w io.Writer) (err error) { + buf := make([]byte, 4) + // See RFC 6637, Section 9, Algorithm-Specific Fields for ECDH keys. + buf[0] = byte(0x03) // Length of the following fields + buf[1] = byte(0x01) // Reserved for future extensions, must be 1 for now + buf[2] = byte(f.KdfHash) + buf[3] = byte(f.KdfAlgo) + _, err = w.Write(buf[:]) + return +} + +func (f *ecdhKdf) byteLen() int { + return 4 +} + +// PublicKey represents an OpenPGP public key. See RFC 4880, section 5.5.2. +type PublicKey struct { + CreationTime time.Time + PubKeyAlgo PublicKeyAlgorithm + PublicKey interface{} // *rsa.PublicKey, *dsa.PublicKey or *ecdsa.PublicKey + Fingerprint [20]byte + KeyId uint64 + IsSubkey bool + + n, e, p, q, g, y parsedMPI + + // RFC 6637 fields + ec *ecdsaKey + ecdh *ecdhKdf +} + +// signingKey provides a convenient abstraction over signature verification +// for v3 and v4 public keys. +type signingKey interface { + SerializeSignaturePrefix(io.Writer) + serializeWithoutHeaders(io.Writer) error +} + +func fromBig(n *big.Int) parsedMPI { + return parsedMPI{ + bytes: n.Bytes(), + bitLength: uint16(n.BitLen()), + } +} + +// NewRSAPublicKey returns a PublicKey that wraps the given rsa.PublicKey. +func NewRSAPublicKey(creationTime time.Time, pub *rsa.PublicKey) *PublicKey { + pk := &PublicKey{ + CreationTime: creationTime, + PubKeyAlgo: PubKeyAlgoRSA, + PublicKey: pub, + n: fromBig(pub.N), + e: fromBig(big.NewInt(int64(pub.E))), + } + + pk.setFingerPrintAndKeyId() + return pk +} + +// NewDSAPublicKey returns a PublicKey that wraps the given rsa.PublicKey. +func NewDSAPublicKey(creationTime time.Time, pub *dsa.PublicKey) *PublicKey { + pk := &PublicKey{ + CreationTime: creationTime, + PubKeyAlgo: PubKeyAlgoDSA, + PublicKey: pub, + p: fromBig(pub.P), + q: fromBig(pub.Q), + g: fromBig(pub.G), + y: fromBig(pub.Y), + } + + pk.setFingerPrintAndKeyId() + return pk +} + +func (pk *PublicKey) parse(r io.Reader) (err error) { + // RFC 4880, section 5.5.2 + var buf [6]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != 4 { + return errors.UnsupportedError("public key version") + } + pk.CreationTime = time.Unix(int64(uint32(buf[1])<<24|uint32(buf[2])<<16|uint32(buf[3])<<8|uint32(buf[4])), 0) + pk.PubKeyAlgo = PublicKeyAlgorithm(buf[5]) + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + err = pk.parseRSA(r) + case PubKeyAlgoDSA: + err = pk.parseDSA(r) + case PubKeyAlgoElGamal: + err = pk.parseElGamal(r) + case PubKeyAlgoECDSA: + pk.ec = new(ecdsaKey) + if err = pk.ec.parse(r); err != nil { + return err + } + pk.PublicKey, err = pk.ec.newECDSA() + case PubKeyAlgoECDH: + pk.ec = new(ecdsaKey) + if err = pk.ec.parse(r); err != nil { + return + } + pk.ecdh = new(ecdhKdf) + if err = pk.ecdh.parse(r); err != nil { + return + } + // The ECDH key is stored in an ecdsa.PublicKey for convenience. + pk.PublicKey, err = pk.ec.newECDSA() + default: + err = errors.UnsupportedError("public key type: " + strconv.Itoa(int(pk.PubKeyAlgo))) + } + if err != nil { + return + } + + pk.setFingerPrintAndKeyId() + return +} + +func (pk *PublicKey) setFingerPrintAndKeyId() { + // RFC 4880, section 12.2 + fingerPrint := sha1.New() + pk.SerializeSignaturePrefix(fingerPrint) + pk.serializeWithoutHeaders(fingerPrint) + copy(pk.Fingerprint[:], fingerPrint.Sum(nil)) + pk.KeyId = binary.BigEndian.Uint64(pk.Fingerprint[12:20]) +} + +// parseRSA parses RSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseRSA(r io.Reader) (err error) { + pk.n.bytes, pk.n.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.e.bytes, pk.e.bitLength, err = readMPI(r) + if err != nil { + return + } + + if len(pk.e.bytes) > 3 { + err = errors.UnsupportedError("large public exponent") + return + } + rsa := &rsa.PublicKey{ + N: new(big.Int).SetBytes(pk.n.bytes), + E: 0, + } + for i := 0; i < len(pk.e.bytes); i++ { + rsa.E <<= 8 + rsa.E |= int(pk.e.bytes[i]) + } + pk.PublicKey = rsa + return +} + +// parseDSA parses DSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseDSA(r io.Reader) (err error) { + pk.p.bytes, pk.p.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.q.bytes, pk.q.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.g.bytes, pk.g.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.y.bytes, pk.y.bitLength, err = readMPI(r) + if err != nil { + return + } + + dsa := new(dsa.PublicKey) + dsa.P = new(big.Int).SetBytes(pk.p.bytes) + dsa.Q = new(big.Int).SetBytes(pk.q.bytes) + dsa.G = new(big.Int).SetBytes(pk.g.bytes) + dsa.Y = new(big.Int).SetBytes(pk.y.bytes) + pk.PublicKey = dsa + return +} + +// parseElGamal parses ElGamal public key material from the given Reader. See +// RFC 4880, section 5.5.2. +func (pk *PublicKey) parseElGamal(r io.Reader) (err error) { + pk.p.bytes, pk.p.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.g.bytes, pk.g.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.y.bytes, pk.y.bitLength, err = readMPI(r) + if err != nil { + return + } + + elgamal := new(elgamal.PublicKey) + elgamal.P = new(big.Int).SetBytes(pk.p.bytes) + elgamal.G = new(big.Int).SetBytes(pk.g.bytes) + elgamal.Y = new(big.Int).SetBytes(pk.y.bytes) + pk.PublicKey = elgamal + return +} + +// SerializeSignaturePrefix writes the prefix for this public key to the given Writer. +// The prefix is used when calculating a signature over this public key. See +// RFC 4880, section 5.2.4. +func (pk *PublicKey) SerializeSignaturePrefix(h io.Writer) { + var pLength uint16 + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + pLength += 2 + uint16(len(pk.n.bytes)) + pLength += 2 + uint16(len(pk.e.bytes)) + case PubKeyAlgoDSA: + pLength += 2 + uint16(len(pk.p.bytes)) + pLength += 2 + uint16(len(pk.q.bytes)) + pLength += 2 + uint16(len(pk.g.bytes)) + pLength += 2 + uint16(len(pk.y.bytes)) + case PubKeyAlgoElGamal: + pLength += 2 + uint16(len(pk.p.bytes)) + pLength += 2 + uint16(len(pk.g.bytes)) + pLength += 2 + uint16(len(pk.y.bytes)) + case PubKeyAlgoECDSA: + pLength += uint16(pk.ec.byteLen()) + case PubKeyAlgoECDH: + pLength += uint16(pk.ec.byteLen()) + pLength += uint16(pk.ecdh.byteLen()) + default: + panic("unknown public key algorithm") + } + pLength += 6 + h.Write([]byte{0x99, byte(pLength >> 8), byte(pLength)}) + return +} + +func (pk *PublicKey) Serialize(w io.Writer) (err error) { + length := 6 // 6 byte header + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + length += 2 + len(pk.n.bytes) + length += 2 + len(pk.e.bytes) + case PubKeyAlgoDSA: + length += 2 + len(pk.p.bytes) + length += 2 + len(pk.q.bytes) + length += 2 + len(pk.g.bytes) + length += 2 + len(pk.y.bytes) + case PubKeyAlgoElGamal: + length += 2 + len(pk.p.bytes) + length += 2 + len(pk.g.bytes) + length += 2 + len(pk.y.bytes) + case PubKeyAlgoECDSA: + length += pk.ec.byteLen() + case PubKeyAlgoECDH: + length += pk.ec.byteLen() + length += pk.ecdh.byteLen() + default: + panic("unknown public key algorithm") + } + + packetType := packetTypePublicKey + if pk.IsSubkey { + packetType = packetTypePublicSubkey + } + err = serializeHeader(w, packetType, length) + if err != nil { + return + } + return pk.serializeWithoutHeaders(w) +} + +// serializeWithoutHeaders marshals the PublicKey to w in the form of an +// OpenPGP public key packet, not including the packet header. +func (pk *PublicKey) serializeWithoutHeaders(w io.Writer) (err error) { + var buf [6]byte + buf[0] = 4 + t := uint32(pk.CreationTime.Unix()) + buf[1] = byte(t >> 24) + buf[2] = byte(t >> 16) + buf[3] = byte(t >> 8) + buf[4] = byte(t) + buf[5] = byte(pk.PubKeyAlgo) + + _, err = w.Write(buf[:]) + if err != nil { + return + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + return writeMPIs(w, pk.n, pk.e) + case PubKeyAlgoDSA: + return writeMPIs(w, pk.p, pk.q, pk.g, pk.y) + case PubKeyAlgoElGamal: + return writeMPIs(w, pk.p, pk.g, pk.y) + case PubKeyAlgoECDSA: + return pk.ec.serialize(w) + case PubKeyAlgoECDH: + if err = pk.ec.serialize(w); err != nil { + return + } + return pk.ecdh.serialize(w) + } + return errors.InvalidArgumentError("bad public-key algorithm") +} + +// CanSign returns true iff this public key can generate signatures +func (pk *PublicKey) CanSign() bool { + return pk.PubKeyAlgo != PubKeyAlgoRSAEncryptOnly && pk.PubKeyAlgo != PubKeyAlgoElGamal +} + +// VerifySignature returns nil iff sig is a valid signature, made by this +// public key, of the data hashed into signed. signed is mutated by this call. +func (pk *PublicKey) VerifySignature(signed hash.Hash, sig *Signature) (err error) { + if !pk.CanSign() { + return errors.InvalidArgumentError("public key cannot generate signatures") + } + + signed.Write(sig.HashSuffix) + hashBytes := signed.Sum(nil) + + if hashBytes[0] != sig.HashTag[0] || hashBytes[1] != sig.HashTag[1] { + return errors.SignatureError("hash tag doesn't match") + } + + if pk.PubKeyAlgo != sig.PubKeyAlgo { + return errors.InvalidArgumentError("public key and signature use different algorithms") + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + rsaPublicKey, _ := pk.PublicKey.(*rsa.PublicKey) + err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.RSASignature.bytes) + if err != nil { + return errors.SignatureError("RSA verification failure") + } + return nil + case PubKeyAlgoDSA: + dsaPublicKey, _ := pk.PublicKey.(*dsa.PublicKey) + // Need to truncate hashBytes to match FIPS 186-3 section 4.6. + subgroupSize := (dsaPublicKey.Q.BitLen() + 7) / 8 + if len(hashBytes) > subgroupSize { + hashBytes = hashBytes[:subgroupSize] + } + if !dsa.Verify(dsaPublicKey, hashBytes, new(big.Int).SetBytes(sig.DSASigR.bytes), new(big.Int).SetBytes(sig.DSASigS.bytes)) { + return errors.SignatureError("DSA verification failure") + } + return nil + case PubKeyAlgoECDSA: + ecdsaPublicKey := pk.PublicKey.(*ecdsa.PublicKey) + if !ecdsa.Verify(ecdsaPublicKey, hashBytes, new(big.Int).SetBytes(sig.ECDSASigR.bytes), new(big.Int).SetBytes(sig.ECDSASigS.bytes)) { + return errors.SignatureError("ECDSA verification failure") + } + return nil + default: + return errors.SignatureError("Unsupported public key algorithm used in signature") + } + panic("unreachable") +} + +// VerifySignatureV3 returns nil iff sig is a valid signature, made by this +// public key, of the data hashed into signed. signed is mutated by this call. +func (pk *PublicKey) VerifySignatureV3(signed hash.Hash, sig *SignatureV3) (err error) { + if !pk.CanSign() { + return errors.InvalidArgumentError("public key cannot generate signatures") + } + + suffix := make([]byte, 5) + suffix[0] = byte(sig.SigType) + binary.BigEndian.PutUint32(suffix[1:], uint32(sig.CreationTime.Unix())) + signed.Write(suffix) + hashBytes := signed.Sum(nil) + + if hashBytes[0] != sig.HashTag[0] || hashBytes[1] != sig.HashTag[1] { + return errors.SignatureError("hash tag doesn't match") + } + + if pk.PubKeyAlgo != sig.PubKeyAlgo { + return errors.InvalidArgumentError("public key and signature use different algorithms") + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + rsaPublicKey := pk.PublicKey.(*rsa.PublicKey) + if err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.RSASignature.bytes); err != nil { + return errors.SignatureError("RSA verification failure") + } + return + case PubKeyAlgoDSA: + dsaPublicKey := pk.PublicKey.(*dsa.PublicKey) + // Need to truncate hashBytes to match FIPS 186-3 section 4.6. + subgroupSize := (dsaPublicKey.Q.BitLen() + 7) / 8 + if len(hashBytes) > subgroupSize { + hashBytes = hashBytes[:subgroupSize] + } + if !dsa.Verify(dsaPublicKey, hashBytes, new(big.Int).SetBytes(sig.DSASigR.bytes), new(big.Int).SetBytes(sig.DSASigS.bytes)) { + return errors.SignatureError("DSA verification failure") + } + return nil + default: + panic("shouldn't happen") + } + panic("unreachable") +} + +// keySignatureHash returns a Hash of the message that needs to be signed for +// pk to assert a subkey relationship to signed. +func keySignatureHash(pk, signed signingKey, hashFunc crypto.Hash) (h hash.Hash, err error) { + if !hashFunc.Available() { + return nil, errors.UnsupportedError("hash function") + } + h = hashFunc.New() + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + signed.SerializeSignaturePrefix(h) + signed.serializeWithoutHeaders(h) + return +} + +// VerifyKeySignature returns nil iff sig is a valid signature, made by this +// public key, of signed. +func (pk *PublicKey) VerifyKeySignature(signed *PublicKey, sig *Signature) (err error) { + h, err := keySignatureHash(pk, signed, sig.Hash) + if err != nil { + return err + } + return pk.VerifySignature(h, sig) +} + +func keyRevocationHash(pk signingKey, hashFunc crypto.Hash) (h hash.Hash, err error) { + if !hashFunc.Available() { + return nil, errors.UnsupportedError("hash function") + } + h = hashFunc.New() + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + + return +} + +// VerifyRevocationSignature returns nil iff sig is a valid signature, made by this +// public key. +func (pk *PublicKey) VerifyRevocationSignature(sig *Signature) (err error) { + h, err := keyRevocationHash(pk, sig.Hash) + if err != nil { + return err + } + return pk.VerifySignature(h, sig) +} + +// userIdSignatureHash returns a Hash of the message that needs to be signed +// to assert that pk is a valid key for id. +func userIdSignatureHash(id string, pk *PublicKey, hashFunc crypto.Hash) (h hash.Hash, err error) { + if !hashFunc.Available() { + return nil, errors.UnsupportedError("hash function") + } + h = hashFunc.New() + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + + var buf [5]byte + buf[0] = 0xb4 + buf[1] = byte(len(id) >> 24) + buf[2] = byte(len(id) >> 16) + buf[3] = byte(len(id) >> 8) + buf[4] = byte(len(id)) + h.Write(buf[:]) + h.Write([]byte(id)) + + return +} + +// VerifyUserIdSignature returns nil iff sig is a valid signature, made by this +// public key, that id is the identity of pub. +func (pk *PublicKey) VerifyUserIdSignature(id string, pub *PublicKey, sig *Signature) (err error) { + h, err := userIdSignatureHash(id, pub, sig.Hash) + if err != nil { + return err + } + return pk.VerifySignature(h, sig) +} + +// VerifyUserIdSignatureV3 returns nil iff sig is a valid signature, made by this +// public key, that id is the identity of pub. +func (pk *PublicKey) VerifyUserIdSignatureV3(id string, pub *PublicKey, sig *SignatureV3) (err error) { + h, err := userIdSignatureV3Hash(id, pub, sig.Hash) + if err != nil { + return err + } + return pk.VerifySignatureV3(h, sig) +} + +// KeyIdString returns the public key's fingerprint in capital hex +// (e.g. "6C7EE1B8621CC013"). +func (pk *PublicKey) KeyIdString() string { + return fmt.Sprintf("%X", pk.Fingerprint[12:20]) +} + +// KeyIdShortString returns the short form of public key's fingerprint +// in capital hex, as shown by gpg --list-keys (e.g. "621CC013"). +func (pk *PublicKey) KeyIdShortString() string { + return fmt.Sprintf("%X", pk.Fingerprint[16:20]) +} + +// A parsedMPI is used to store the contents of a big integer, along with the +// bit length that was specified in the original input. This allows the MPI to +// be reserialized exactly. +type parsedMPI struct { + bytes []byte + bitLength uint16 +} + +// writeMPIs is a utility function for serializing several big integers to the +// given Writer. +func writeMPIs(w io.Writer, mpis ...parsedMPI) (err error) { + for _, mpi := range mpis { + err = writeMPI(w, mpi.bitLength, mpi.bytes) + if err != nil { + return + } + } + return +} + +// BitLength returns the bit length for the given public key. +func (pk *PublicKey) BitLength() (bitLength uint16, err error) { + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + bitLength = pk.n.bitLength + case PubKeyAlgoDSA: + bitLength = pk.p.bitLength + case PubKeyAlgoElGamal: + bitLength = pk.p.bitLength + default: + err = errors.InvalidArgumentError("bad public-key algorithm") + } + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_test.go new file mode 100644 index 00000000000..7ad7d91856d --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_test.go @@ -0,0 +1,202 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "testing" + "time" +) + +var pubKeyTests = []struct { + hexData string + hexFingerprint string + creationTime time.Time + pubKeyAlgo PublicKeyAlgorithm + keyId uint64 + keyIdString string + keyIdShort string +}{ + {rsaPkDataHex, rsaFingerprintHex, time.Unix(0x4d3c5c10, 0), PubKeyAlgoRSA, 0xa34d7e18c20c31bb, "A34D7E18C20C31BB", "C20C31BB"}, + {dsaPkDataHex, dsaFingerprintHex, time.Unix(0x4d432f89, 0), PubKeyAlgoDSA, 0x8e8fbe54062f19ed, "8E8FBE54062F19ED", "062F19ED"}, + {ecdsaPkDataHex, ecdsaFingerprintHex, time.Unix(0x5071c294, 0), PubKeyAlgoECDSA, 0x43fe956c542ca00b, "43FE956C542CA00B", "542CA00B"}, +} + +func TestPublicKeyRead(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + continue + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + continue + } + if pk.PubKeyAlgo != test.pubKeyAlgo { + t.Errorf("#%d: bad public key algorithm got:%x want:%x", i, pk.PubKeyAlgo, test.pubKeyAlgo) + } + if !pk.CreationTime.Equal(test.creationTime) { + t.Errorf("#%d: bad creation time got:%v want:%v", i, pk.CreationTime, test.creationTime) + } + expectedFingerprint, _ := hex.DecodeString(test.hexFingerprint) + if !bytes.Equal(expectedFingerprint, pk.Fingerprint[:]) { + t.Errorf("#%d: bad fingerprint got:%x want:%x", i, pk.Fingerprint[:], expectedFingerprint) + } + if pk.KeyId != test.keyId { + t.Errorf("#%d: bad keyid got:%x want:%x", i, pk.KeyId, test.keyId) + } + if g, e := pk.KeyIdString(), test.keyIdString; g != e { + t.Errorf("#%d: bad KeyIdString got:%q want:%q", i, g, e) + } + if g, e := pk.KeyIdShortString(), test.keyIdShort; g != e { + t.Errorf("#%d: bad KeyIdShortString got:%q want:%q", i, g, e) + } + } +} + +func TestPublicKeySerialize(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + continue + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + continue + } + serializeBuf := bytes.NewBuffer(nil) + err = pk.Serialize(serializeBuf) + if err != nil { + t.Errorf("#%d: failed to serialize: %s", i, err) + continue + } + + packet, err = Read(serializeBuf) + if err != nil { + t.Errorf("#%d: Read error (from serialized data): %s", i, err) + continue + } + pk, ok = packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse serialized data, got: %#v", i, packet) + continue + } + } +} + +func TestEcc384Serialize(t *testing.T) { + r := readerFromHex(ecc384PubHex) + var w bytes.Buffer + for i := 0; i < 2; i++ { + // Public key + p, err := Read(r) + if err != nil { + t.Error(err) + } + pubkey := p.(*PublicKey) + if !bytes.Equal(pubkey.ec.oid, []byte{0x2b, 0x81, 0x04, 0x00, 0x22}) { + t.Errorf("Unexpected pubkey OID: %x", pubkey.ec.oid) + } + if !bytes.Equal(pubkey.ec.p.bytes[:5], []byte{0x04, 0xf6, 0xb8, 0xc5, 0xac}) { + t.Errorf("Unexpected pubkey P[:5]: %x", pubkey.ec.p.bytes) + } + if pubkey.KeyId != 0x098033880F54719F { + t.Errorf("Unexpected pubkey ID: %x", pubkey.KeyId) + } + err = pubkey.Serialize(&w) + if err != nil { + t.Error(err) + } + // User ID + p, err = Read(r) + if err != nil { + t.Error(err) + } + uid := p.(*UserId) + if uid.Id != "ec_dsa_dh_384 " { + t.Error("Unexpected UID:", uid.Id) + } + err = uid.Serialize(&w) + if err != nil { + t.Error(err) + } + // User ID Sig + p, err = Read(r) + if err != nil { + t.Error(err) + } + uidSig := p.(*Signature) + err = pubkey.VerifyUserIdSignature(uid.Id, pubkey, uidSig) + if err != nil { + t.Error(err, ": UID") + } + err = uidSig.Serialize(&w) + if err != nil { + t.Error(err) + } + // Subkey + p, err = Read(r) + if err != nil { + t.Error(err) + } + subkey := p.(*PublicKey) + if !bytes.Equal(subkey.ec.oid, []byte{0x2b, 0x81, 0x04, 0x00, 0x22}) { + t.Errorf("Unexpected subkey OID: %x", subkey.ec.oid) + } + if !bytes.Equal(subkey.ec.p.bytes[:5], []byte{0x04, 0x2f, 0xaa, 0x84, 0x02}) { + t.Errorf("Unexpected subkey P[:5]: %x", subkey.ec.p.bytes) + } + if subkey.ecdh.KdfHash != 0x09 { + t.Error("Expected KDF hash function SHA384 (0x09), got", subkey.ecdh.KdfHash) + } + if subkey.ecdh.KdfAlgo != 0x09 { + t.Error("Expected KDF symmetric alg AES256 (0x09), got", subkey.ecdh.KdfAlgo) + } + if subkey.KeyId != 0xAA8B938F9A201946 { + t.Errorf("Unexpected subkey ID: %x", subkey.KeyId) + } + err = subkey.Serialize(&w) + if err != nil { + t.Error(err) + } + // Subkey Sig + p, err = Read(r) + if err != nil { + t.Error(err) + } + subkeySig := p.(*Signature) + err = pubkey.VerifyKeySignature(subkey, subkeySig) + if err != nil { + t.Error(err) + } + err = subkeySig.Serialize(&w) + if err != nil { + t.Error(err) + } + // Now read back what we've written again + r = bytes.NewBuffer(w.Bytes()) + w.Reset() + } +} + +const rsaFingerprintHex = "5fb74b1d03b1e3cb31bc2f8aa34d7e18c20c31bb" + +const rsaPkDataHex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001" + +const dsaFingerprintHex = "eece4c094db002103714c63c8e8fbe54062f19ed" + +const dsaPkDataHex = "9901a2044d432f89110400cd581334f0d7a1e1bdc8b9d6d8c0baf68793632735d2bb0903224cbaa1dfbf35a60ee7a13b92643421e1eb41aa8d79bea19a115a677f6b8ba3c7818ce53a6c2a24a1608bd8b8d6e55c5090cbde09dd26e356267465ae25e69ec8bdd57c7bbb2623e4d73336f73a0a9098f7f16da2e25252130fd694c0e8070c55a812a423ae7f00a0ebf50e70c2f19c3520a551bd4b08d30f23530d3d03ff7d0bf4a53a64a09dc5e6e6e35854b7d70c882b0c60293401958b1bd9e40abec3ea05ba87cf64899299d4bd6aa7f459c201d3fbbd6c82004bdc5e8a9eb8082d12054cc90fa9d4ec251a843236a588bf49552441817436c4f43326966fe85447d4e6d0acf8fa1ef0f014730770603ad7634c3088dc52501c237328417c31c89ed70400b2f1a98b0bf42f11fefc430704bebbaa41d9f355600c3facee1e490f64208e0e094ea55e3a598a219a58500bf78ac677b670a14f4e47e9cf8eab4f368cc1ddcaa18cc59309d4cc62dd4f680e73e6cc3e1ce87a84d0925efbcb26c575c093fc42eecf45135fabf6403a25c2016e1774c0484e440a18319072c617cc97ac0a3bb0" + +const ecdsaFingerprintHex = "9892270b38b8980b05c8d56d43fe956c542ca00b" + +const ecdsaPkDataHex = "9893045071c29413052b8104002304230401f4867769cedfa52c325018896245443968e52e51d0c2df8d939949cb5b330f2921711fbee1c9b9dddb95d15cb0255e99badeddda7cc23d9ddcaacbc290969b9f24019375d61c2e4e3b36953a28d8b2bc95f78c3f1d592fb24499be348656a7b17e3963187b4361afe497bc5f9f81213f04069f8e1fb9e6a6290ae295ca1a92b894396cb4" + +// Source: https://sites.google.com/site/brainhub/pgpecckeys#TOC-ECC-NIST-P-384-key +const ecc384PubHex = `99006f044d53059213052b81040022030304f6b8c5aced5b84ef9f4a209db2e4a9dfb70d28cb8c10ecd57674a9fa5a67389942b62d5e51367df4c7bfd3f8e500feecf07ed265a621a8ebbbe53e947ec78c677eba143bd1533c2b350e1c29f82313e1e1108eba063be1e64b10e6950e799c2db42465635f6473615f64685f333834203c6f70656e70677040627261696e6875622e6f72673e8900cb04101309005305024d530592301480000000002000077072656665727265642d656d61696c2d656e636f64696e67407067702e636f6d7067706d696d65040b090807021901051b03000000021602051e010000000415090a08000a0910098033880f54719fca2b0180aa37350968bd5f115afd8ce7bc7b103822152dbff06d0afcda835329510905b98cb469ba208faab87c7412b799e7b633017f58364ea480e8a1a3f253a0c5f22c446e8be9a9fce6210136ee30811abbd49139de28b5bdf8dc36d06ae748579e9ff503b90073044d53059212052b810400220303042faa84024a20b6735c4897efa5bfb41bf85b7eefeab5ca0cb9ffc8ea04a46acb25534a577694f9e25340a4ab5223a9dd1eda530c8aa2e6718db10d7e672558c7736fe09369ea5739a2a3554bf16d41faa50562f11c6d39bbd5dffb6b9a9ec9180301090989008404181309000c05024d530592051b0c000000000a0910098033880f54719f80970180eee7a6d8fcee41ee4f9289df17f9bcf9d955dca25c583b94336f3a2b2d4986dc5cf417b8d2dc86f741a9e1a6d236c0e3017d1c76575458a0cfb93ae8a2b274fcc65ceecd7a91eec83656ba13219969f06945b48c56bd04152c3a0553c5f2f4bd1267` diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_v3.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_v3.go new file mode 100644 index 00000000000..164124807b8 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_v3.go @@ -0,0 +1,275 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/md5" + "crypto/rsa" + "encoding/binary" + "fmt" + "hash" + "io" + "math/big" + "strconv" + "time" + + "golang.org/x/crypto/openpgp/errors" +) + +// PublicKeyV3 represents older, version 3 public keys. These keys are less secure and +// should not be used for signing or encrypting. They are supported here only for +// parsing version 3 key material and validating signatures. +// See RFC 4880, section 5.5.2. +type PublicKeyV3 struct { + CreationTime time.Time + DaysToExpire uint16 + PubKeyAlgo PublicKeyAlgorithm + PublicKey *rsa.PublicKey + Fingerprint [16]byte + KeyId uint64 + IsSubkey bool + + n, e parsedMPI +} + +// newRSAPublicKeyV3 returns a PublicKey that wraps the given rsa.PublicKey. +// Included here for testing purposes only. RFC 4880, section 5.5.2: +// "an implementation MUST NOT generate a V3 key, but MAY accept it." +func newRSAPublicKeyV3(creationTime time.Time, pub *rsa.PublicKey) *PublicKeyV3 { + pk := &PublicKeyV3{ + CreationTime: creationTime, + PublicKey: pub, + n: fromBig(pub.N), + e: fromBig(big.NewInt(int64(pub.E))), + } + + pk.setFingerPrintAndKeyId() + return pk +} + +func (pk *PublicKeyV3) parse(r io.Reader) (err error) { + // RFC 4880, section 5.5.2 + var buf [8]byte + if _, err = readFull(r, buf[:]); err != nil { + return + } + if buf[0] < 2 || buf[0] > 3 { + return errors.UnsupportedError("public key version") + } + pk.CreationTime = time.Unix(int64(uint32(buf[1])<<24|uint32(buf[2])<<16|uint32(buf[3])<<8|uint32(buf[4])), 0) + pk.DaysToExpire = binary.BigEndian.Uint16(buf[5:7]) + pk.PubKeyAlgo = PublicKeyAlgorithm(buf[7]) + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + err = pk.parseRSA(r) + default: + err = errors.UnsupportedError("public key type: " + strconv.Itoa(int(pk.PubKeyAlgo))) + } + if err != nil { + return + } + + pk.setFingerPrintAndKeyId() + return +} + +func (pk *PublicKeyV3) setFingerPrintAndKeyId() { + // RFC 4880, section 12.2 + fingerPrint := md5.New() + fingerPrint.Write(pk.n.bytes) + fingerPrint.Write(pk.e.bytes) + fingerPrint.Sum(pk.Fingerprint[:0]) + pk.KeyId = binary.BigEndian.Uint64(pk.n.bytes[len(pk.n.bytes)-8:]) +} + +// parseRSA parses RSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKeyV3) parseRSA(r io.Reader) (err error) { + if pk.n.bytes, pk.n.bitLength, err = readMPI(r); err != nil { + return + } + if pk.e.bytes, pk.e.bitLength, err = readMPI(r); err != nil { + return + } + + if len(pk.e.bytes) > 3 { + err = errors.UnsupportedError("large public exponent") + return + } + rsa := &rsa.PublicKey{N: new(big.Int).SetBytes(pk.n.bytes)} + for i := 0; i < len(pk.e.bytes); i++ { + rsa.E <<= 8 + rsa.E |= int(pk.e.bytes[i]) + } + pk.PublicKey = rsa + return +} + +// SerializeSignaturePrefix writes the prefix for this public key to the given Writer. +// The prefix is used when calculating a signature over this public key. See +// RFC 4880, section 5.2.4. +func (pk *PublicKeyV3) SerializeSignaturePrefix(w io.Writer) { + var pLength uint16 + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + pLength += 2 + uint16(len(pk.n.bytes)) + pLength += 2 + uint16(len(pk.e.bytes)) + default: + panic("unknown public key algorithm") + } + pLength += 6 + w.Write([]byte{0x99, byte(pLength >> 8), byte(pLength)}) + return +} + +func (pk *PublicKeyV3) Serialize(w io.Writer) (err error) { + length := 8 // 8 byte header + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + length += 2 + len(pk.n.bytes) + length += 2 + len(pk.e.bytes) + default: + panic("unknown public key algorithm") + } + + packetType := packetTypePublicKey + if pk.IsSubkey { + packetType = packetTypePublicSubkey + } + if err = serializeHeader(w, packetType, length); err != nil { + return + } + return pk.serializeWithoutHeaders(w) +} + +// serializeWithoutHeaders marshals the PublicKey to w in the form of an +// OpenPGP public key packet, not including the packet header. +func (pk *PublicKeyV3) serializeWithoutHeaders(w io.Writer) (err error) { + var buf [8]byte + // Version 3 + buf[0] = 3 + // Creation time + t := uint32(pk.CreationTime.Unix()) + buf[1] = byte(t >> 24) + buf[2] = byte(t >> 16) + buf[3] = byte(t >> 8) + buf[4] = byte(t) + // Days to expire + buf[5] = byte(pk.DaysToExpire >> 8) + buf[6] = byte(pk.DaysToExpire) + // Public key algorithm + buf[7] = byte(pk.PubKeyAlgo) + + if _, err = w.Write(buf[:]); err != nil { + return + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + return writeMPIs(w, pk.n, pk.e) + } + return errors.InvalidArgumentError("bad public-key algorithm") +} + +// CanSign returns true iff this public key can generate signatures +func (pk *PublicKeyV3) CanSign() bool { + return pk.PubKeyAlgo != PubKeyAlgoRSAEncryptOnly +} + +// VerifySignatureV3 returns nil iff sig is a valid signature, made by this +// public key, of the data hashed into signed. signed is mutated by this call. +func (pk *PublicKeyV3) VerifySignatureV3(signed hash.Hash, sig *SignatureV3) (err error) { + if !pk.CanSign() { + return errors.InvalidArgumentError("public key cannot generate signatures") + } + + suffix := make([]byte, 5) + suffix[0] = byte(sig.SigType) + binary.BigEndian.PutUint32(suffix[1:], uint32(sig.CreationTime.Unix())) + signed.Write(suffix) + hashBytes := signed.Sum(nil) + + if hashBytes[0] != sig.HashTag[0] || hashBytes[1] != sig.HashTag[1] { + return errors.SignatureError("hash tag doesn't match") + } + + if pk.PubKeyAlgo != sig.PubKeyAlgo { + return errors.InvalidArgumentError("public key and signature use different algorithms") + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + if err = rsa.VerifyPKCS1v15(pk.PublicKey, sig.Hash, hashBytes, sig.RSASignature.bytes); err != nil { + return errors.SignatureError("RSA verification failure") + } + return + default: + // V3 public keys only support RSA. + panic("shouldn't happen") + } + panic("unreachable") +} + +// VerifyUserIdSignatureV3 returns nil iff sig is a valid signature, made by this +// public key, that id is the identity of pub. +func (pk *PublicKeyV3) VerifyUserIdSignatureV3(id string, pub *PublicKeyV3, sig *SignatureV3) (err error) { + h, err := userIdSignatureV3Hash(id, pk, sig.Hash) + if err != nil { + return err + } + return pk.VerifySignatureV3(h, sig) +} + +// VerifyKeySignatureV3 returns nil iff sig is a valid signature, made by this +// public key, of signed. +func (pk *PublicKeyV3) VerifyKeySignatureV3(signed *PublicKeyV3, sig *SignatureV3) (err error) { + h, err := keySignatureHash(pk, signed, sig.Hash) + if err != nil { + return err + } + return pk.VerifySignatureV3(h, sig) +} + +// userIdSignatureV3Hash returns a Hash of the message that needs to be signed +// to assert that pk is a valid key for id. +func userIdSignatureV3Hash(id string, pk signingKey, hfn crypto.Hash) (h hash.Hash, err error) { + if !hfn.Available() { + return nil, errors.UnsupportedError("hash function") + } + h = hfn.New() + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + + h.Write([]byte(id)) + + return +} + +// KeyIdString returns the public key's fingerprint in capital hex +// (e.g. "6C7EE1B8621CC013"). +func (pk *PublicKeyV3) KeyIdString() string { + return fmt.Sprintf("%X", pk.KeyId) +} + +// KeyIdShortString returns the short form of public key's fingerprint +// in capital hex, as shown by gpg --list-keys (e.g. "621CC013"). +func (pk *PublicKeyV3) KeyIdShortString() string { + return fmt.Sprintf("%X", pk.KeyId&0xFFFFFFFF) +} + +// BitLength returns the bit length for the given public key. +func (pk *PublicKeyV3) BitLength() (bitLength uint16, err error) { + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + bitLength = pk.n.bitLength + default: + err = errors.InvalidArgumentError("bad public-key algorithm") + } + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_v3_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_v3_test.go new file mode 100644 index 00000000000..e06405904b3 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/public_key_v3_test.go @@ -0,0 +1,82 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "testing" + "time" +) + +var pubKeyV3Test = struct { + hexFingerprint string + creationTime time.Time + pubKeyAlgo PublicKeyAlgorithm + keyId uint64 + keyIdString string + keyIdShort string +}{ + "103BECF5BD1E837C89D19E98487767F7", + time.Unix(779753634, 0), + PubKeyAlgoRSA, + 0xDE0F188A5DA5E3C9, + "DE0F188A5DA5E3C9", + "5DA5E3C9"} + +func TestPublicKeyV3Read(t *testing.T) { + i, test := 0, pubKeyV3Test + packet, err := Read(v3KeyReader(t)) + if err != nil { + t.Fatalf("#%d: Read error: %s", i, err) + } + pk, ok := packet.(*PublicKeyV3) + if !ok { + t.Fatalf("#%d: failed to parse, got: %#v", i, packet) + } + if pk.PubKeyAlgo != test.pubKeyAlgo { + t.Errorf("#%d: bad public key algorithm got:%x want:%x", i, pk.PubKeyAlgo, test.pubKeyAlgo) + } + if !pk.CreationTime.Equal(test.creationTime) { + t.Errorf("#%d: bad creation time got:%v want:%v", i, pk.CreationTime, test.creationTime) + } + expectedFingerprint, _ := hex.DecodeString(test.hexFingerprint) + if !bytes.Equal(expectedFingerprint, pk.Fingerprint[:]) { + t.Errorf("#%d: bad fingerprint got:%x want:%x", i, pk.Fingerprint[:], expectedFingerprint) + } + if pk.KeyId != test.keyId { + t.Errorf("#%d: bad keyid got:%x want:%x", i, pk.KeyId, test.keyId) + } + if g, e := pk.KeyIdString(), test.keyIdString; g != e { + t.Errorf("#%d: bad KeyIdString got:%q want:%q", i, g, e) + } + if g, e := pk.KeyIdShortString(), test.keyIdShort; g != e { + t.Errorf("#%d: bad KeyIdShortString got:%q want:%q", i, g, e) + } +} + +func TestPublicKeyV3Serialize(t *testing.T) { + //for i, test := range pubKeyV3Tests { + i := 0 + packet, err := Read(v3KeyReader(t)) + if err != nil { + t.Fatalf("#%d: Read error: %s", i, err) + } + pk, ok := packet.(*PublicKeyV3) + if !ok { + t.Fatalf("#%d: failed to parse, got: %#v", i, packet) + } + var serializeBuf bytes.Buffer + if err = pk.Serialize(&serializeBuf); err != nil { + t.Fatalf("#%d: failed to serialize: %s", i, err) + } + + if packet, err = Read(bytes.NewBuffer(serializeBuf.Bytes())); err != nil { + t.Fatalf("#%d: Read error (from serialized data): %s", i, err) + } + if pk, ok = packet.(*PublicKeyV3); !ok { + t.Fatalf("#%d: failed to parse serialized data, got: %#v", i, packet) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/reader.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/reader.go new file mode 100644 index 00000000000..753bf8f6603 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/reader.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "golang.org/x/crypto/openpgp/errors" + "io" +) + +// Reader reads packets from an io.Reader and allows packets to be 'unread' so +// that they result from the next call to Next. +type Reader struct { + q []Packet + readers []io.Reader +} + +// Next returns the most recently unread Packet, or reads another packet from +// the top-most io.Reader. Unknown packet types are skipped. +func (r *Reader) Next() (p Packet, err error) { + if len(r.q) > 0 { + p = r.q[len(r.q)-1] + r.q = r.q[:len(r.q)-1] + return + } + + for len(r.readers) > 0 { + p, err = Read(r.readers[len(r.readers)-1]) + if err == nil { + return + } + if err == io.EOF { + r.readers = r.readers[:len(r.readers)-1] + continue + } + if _, ok := err.(errors.UnknownPacketTypeError); !ok { + return nil, err + } + } + + return nil, io.EOF +} + +// Push causes the Reader to start reading from a new io.Reader. When an EOF +// error is seen from the new io.Reader, it is popped and the Reader continues +// to read from the next most recent io.Reader. +func (r *Reader) Push(reader io.Reader) { + r.readers = append(r.readers, reader) +} + +// Unread causes the given Packet to be returned from the next call to Next. +func (r *Reader) Unread(p Packet) { + r.q = append(r.q, p) +} + +func NewReader(r io.Reader) *Reader { + return &Reader{ + q: nil, + readers: []io.Reader{r}, + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature.go new file mode 100644 index 00000000000..79cf993f579 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature.go @@ -0,0 +1,663 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/dsa" + "crypto/rsa" + "encoding/binary" + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/s2k" + "hash" + "io" + "strconv" + "time" +) + +const ( + // See RFC 4880, section 5.2.3.21 for details. + KeyFlagCertify = 1 << iota + KeyFlagSign + KeyFlagEncryptCommunications + KeyFlagEncryptStorage +) + +// Signature represents a signature. See RFC 4880, section 5.2. +type Signature struct { + SigType SignatureType + PubKeyAlgo PublicKeyAlgorithm + Hash crypto.Hash + + // HashSuffix is extra data that is hashed in after the signed data. + HashSuffix []byte + // HashTag contains the first two bytes of the hash for fast rejection + // of bad signed data. + HashTag [2]byte + CreationTime time.Time + + RSASignature parsedMPI + DSASigR, DSASigS parsedMPI + ECDSASigR, ECDSASigS parsedMPI + + // rawSubpackets contains the unparsed subpackets, in order. + rawSubpackets []outputSubpacket + + // The following are optional so are nil when not included in the + // signature. + + SigLifetimeSecs, KeyLifetimeSecs *uint32 + PreferredSymmetric, PreferredHash, PreferredCompression []uint8 + IssuerKeyId *uint64 + IsPrimaryId *bool + + // FlagsValid is set if any flags were given. See RFC 4880, section + // 5.2.3.21 for details. + FlagsValid bool + FlagCertify, FlagSign, FlagEncryptCommunications, FlagEncryptStorage bool + + // RevocationReason is set if this signature has been revoked. + // See RFC 4880, section 5.2.3.23 for details. + RevocationReason *uint8 + RevocationReasonText string + + outSubpackets []outputSubpacket +} + +func (sig *Signature) parse(r io.Reader) (err error) { + // RFC 4880, section 5.2.3 + var buf [5]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + if buf[0] != 4 { + err = errors.UnsupportedError("signature packet version " + strconv.Itoa(int(buf[0]))) + return + } + + _, err = readFull(r, buf[:5]) + if err != nil { + return + } + sig.SigType = SignatureType(buf[0]) + sig.PubKeyAlgo = PublicKeyAlgorithm(buf[1]) + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoDSA, PubKeyAlgoECDSA: + default: + err = errors.UnsupportedError("public key algorithm " + strconv.Itoa(int(sig.PubKeyAlgo))) + return + } + + var ok bool + sig.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return errors.UnsupportedError("hash function " + strconv.Itoa(int(buf[2]))) + } + + hashedSubpacketsLength := int(buf[3])<<8 | int(buf[4]) + l := 6 + hashedSubpacketsLength + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + copy(sig.HashSuffix[1:], buf[:5]) + hashedSubpackets := sig.HashSuffix[6:l] + _, err = readFull(r, hashedSubpackets) + if err != nil { + return + } + // See RFC 4880, section 5.2.4 + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = uint8(l >> 24) + trailer[3] = uint8(l >> 16) + trailer[4] = uint8(l >> 8) + trailer[5] = uint8(l) + + err = parseSignatureSubpackets(sig, hashedSubpackets, true) + if err != nil { + return + } + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + unhashedSubpacketsLength := int(buf[0])<<8 | int(buf[1]) + unhashedSubpackets := make([]byte, unhashedSubpacketsLength) + _, err = readFull(r, unhashedSubpackets) + if err != nil { + return + } + err = parseSignatureSubpackets(sig, unhashedSubpackets, false) + if err != nil { + return + } + + _, err = readFull(r, sig.HashTag[:2]) + if err != nil { + return + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature.bytes, sig.RSASignature.bitLength, err = readMPI(r) + case PubKeyAlgoDSA: + sig.DSASigR.bytes, sig.DSASigR.bitLength, err = readMPI(r) + if err == nil { + sig.DSASigS.bytes, sig.DSASigS.bitLength, err = readMPI(r) + } + case PubKeyAlgoECDSA: + sig.ECDSASigR.bytes, sig.ECDSASigR.bitLength, err = readMPI(r) + if err == nil { + sig.ECDSASigS.bytes, sig.ECDSASigS.bitLength, err = readMPI(r) + } + default: + panic("unreachable") + } + return +} + +// parseSignatureSubpackets parses subpackets of the main signature packet. See +// RFC 4880, section 5.2.3.1. +func parseSignatureSubpackets(sig *Signature, subpackets []byte, isHashed bool) (err error) { + for len(subpackets) > 0 { + subpackets, err = parseSignatureSubpacket(sig, subpackets, isHashed) + if err != nil { + return + } + } + + if sig.CreationTime.IsZero() { + err = errors.StructuralError("no creation time in signature") + } + + return +} + +type signatureSubpacketType uint8 + +const ( + creationTimeSubpacket signatureSubpacketType = 2 + signatureExpirationSubpacket signatureSubpacketType = 3 + keyExpirationSubpacket signatureSubpacketType = 9 + prefSymmetricAlgosSubpacket signatureSubpacketType = 11 + issuerSubpacket signatureSubpacketType = 16 + prefHashAlgosSubpacket signatureSubpacketType = 21 + prefCompressionSubpacket signatureSubpacketType = 22 + primaryUserIdSubpacket signatureSubpacketType = 25 + keyFlagsSubpacket signatureSubpacketType = 27 + reasonForRevocationSubpacket signatureSubpacketType = 29 +) + +// parseSignatureSubpacket parses a single subpacket. len(subpacket) is >= 1. +func parseSignatureSubpacket(sig *Signature, subpacket []byte, isHashed bool) (rest []byte, err error) { + // RFC 4880, section 5.2.3.1 + var ( + length uint32 + packetType signatureSubpacketType + isCritical bool + ) + switch { + case subpacket[0] < 192: + length = uint32(subpacket[0]) + subpacket = subpacket[1:] + case subpacket[0] < 255: + if len(subpacket) < 2 { + goto Truncated + } + length = uint32(subpacket[0]-192)<<8 + uint32(subpacket[1]) + 192 + subpacket = subpacket[2:] + default: + if len(subpacket) < 5 { + goto Truncated + } + length = uint32(subpacket[1])<<24 | + uint32(subpacket[2])<<16 | + uint32(subpacket[3])<<8 | + uint32(subpacket[4]) + subpacket = subpacket[5:] + } + if length > uint32(len(subpacket)) { + goto Truncated + } + rest = subpacket[length:] + subpacket = subpacket[:length] + if len(subpacket) == 0 { + err = errors.StructuralError("zero length signature subpacket") + return + } + packetType = signatureSubpacketType(subpacket[0] & 0x7f) + isCritical = subpacket[0]&0x80 == 0x80 + subpacket = subpacket[1:] + sig.rawSubpackets = append(sig.rawSubpackets, outputSubpacket{isHashed, packetType, isCritical, subpacket}) + switch packetType { + case creationTimeSubpacket: + if !isHashed { + err = errors.StructuralError("signature creation time in non-hashed area") + return + } + if len(subpacket) != 4 { + err = errors.StructuralError("signature creation time not four bytes") + return + } + t := binary.BigEndian.Uint32(subpacket) + sig.CreationTime = time.Unix(int64(t), 0) + case signatureExpirationSubpacket: + // Signature expiration time, section 5.2.3.10 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = errors.StructuralError("expiration subpacket with bad length") + return + } + sig.SigLifetimeSecs = new(uint32) + *sig.SigLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case keyExpirationSubpacket: + // Key expiration time, section 5.2.3.6 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = errors.StructuralError("key expiration subpacket with bad length") + return + } + sig.KeyLifetimeSecs = new(uint32) + *sig.KeyLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case prefSymmetricAlgosSubpacket: + // Preferred symmetric algorithms, section 5.2.3.7 + if !isHashed { + return + } + sig.PreferredSymmetric = make([]byte, len(subpacket)) + copy(sig.PreferredSymmetric, subpacket) + case issuerSubpacket: + // Issuer, section 5.2.3.5 + if len(subpacket) != 8 { + err = errors.StructuralError("issuer subpacket with bad length") + return + } + sig.IssuerKeyId = new(uint64) + *sig.IssuerKeyId = binary.BigEndian.Uint64(subpacket) + case prefHashAlgosSubpacket: + // Preferred hash algorithms, section 5.2.3.8 + if !isHashed { + return + } + sig.PreferredHash = make([]byte, len(subpacket)) + copy(sig.PreferredHash, subpacket) + case prefCompressionSubpacket: + // Preferred compression algorithms, section 5.2.3.9 + if !isHashed { + return + } + sig.PreferredCompression = make([]byte, len(subpacket)) + copy(sig.PreferredCompression, subpacket) + case primaryUserIdSubpacket: + // Primary User ID, section 5.2.3.19 + if !isHashed { + return + } + if len(subpacket) != 1 { + err = errors.StructuralError("primary user id subpacket with bad length") + return + } + sig.IsPrimaryId = new(bool) + if subpacket[0] > 0 { + *sig.IsPrimaryId = true + } + case keyFlagsSubpacket: + // Key flags, section 5.2.3.21 + if !isHashed { + return + } + if len(subpacket) == 0 { + err = errors.StructuralError("empty key flags subpacket") + return + } + sig.FlagsValid = true + if subpacket[0]&KeyFlagCertify != 0 { + sig.FlagCertify = true + } + if subpacket[0]&KeyFlagSign != 0 { + sig.FlagSign = true + } + if subpacket[0]&KeyFlagEncryptCommunications != 0 { + sig.FlagEncryptCommunications = true + } + if subpacket[0]&KeyFlagEncryptStorage != 0 { + sig.FlagEncryptStorage = true + } + case reasonForRevocationSubpacket: + // Reason For Revocation, section 5.2.3.23 + if !isHashed { + return + } + if len(subpacket) == 0 { + err = errors.StructuralError("empty revocation reason subpacket") + return + } + sig.RevocationReason = new(uint8) + *sig.RevocationReason = subpacket[0] + sig.RevocationReasonText = string(subpacket[1:]) + + default: + if isCritical { + err = errors.UnsupportedError("unknown critical signature subpacket type " + strconv.Itoa(int(packetType))) + return + } + } + return + +Truncated: + err = errors.StructuralError("signature subpacket truncated") + return +} + +// subpacketLengthLength returns the length, in bytes, of an encoded length value. +func subpacketLengthLength(length int) int { + if length < 192 { + return 1 + } + if length < 16320 { + return 2 + } + return 5 +} + +// serializeSubpacketLength marshals the given length into to. +func serializeSubpacketLength(to []byte, length int) int { + // RFC 4880, Section 4.2.2. + if length < 192 { + to[0] = byte(length) + return 1 + } + if length < 16320 { + length -= 192 + to[0] = byte((length >> 8) + 192) + to[1] = byte(length) + return 2 + } + to[0] = 255 + to[1] = byte(length >> 24) + to[2] = byte(length >> 16) + to[3] = byte(length >> 8) + to[4] = byte(length) + return 5 +} + +// subpacketsLength returns the serialized length, in bytes, of the given +// subpackets. +func subpacketsLength(subpackets []outputSubpacket, hashed bool) (length int) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + length += subpacketLengthLength(len(subpacket.contents) + 1) + length += 1 // type byte + length += len(subpacket.contents) + } + } + return +} + +// serializeSubpackets marshals the given subpackets into to. +func serializeSubpackets(to []byte, subpackets []outputSubpacket, hashed bool) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + n := serializeSubpacketLength(to, len(subpacket.contents)+1) + to[n] = byte(subpacket.subpacketType) + to = to[1+n:] + n = copy(to, subpacket.contents) + to = to[n:] + } + } + return +} + +// KeyExpired returns whether sig is a self-signature of a key that has +// expired. +func (sig *Signature) KeyExpired(currentTime time.Time) bool { + if sig.KeyLifetimeSecs == nil { + return false + } + expiry := sig.CreationTime.Add(time.Duration(*sig.KeyLifetimeSecs) * time.Second) + return currentTime.After(expiry) +} + +// buildHashSuffix constructs the HashSuffix member of sig in preparation for signing. +func (sig *Signature) buildHashSuffix() (err error) { + hashedSubpacketsLen := subpacketsLength(sig.outSubpackets, true) + + var ok bool + l := 6 + hashedSubpacketsLen + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + sig.HashSuffix[1] = uint8(sig.SigType) + sig.HashSuffix[2] = uint8(sig.PubKeyAlgo) + sig.HashSuffix[3], ok = s2k.HashToHashId(sig.Hash) + if !ok { + sig.HashSuffix = nil + return errors.InvalidArgumentError("hash cannot be represented in OpenPGP: " + strconv.Itoa(int(sig.Hash))) + } + sig.HashSuffix[4] = byte(hashedSubpacketsLen >> 8) + sig.HashSuffix[5] = byte(hashedSubpacketsLen) + serializeSubpackets(sig.HashSuffix[6:l], sig.outSubpackets, true) + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = byte(l >> 24) + trailer[3] = byte(l >> 16) + trailer[4] = byte(l >> 8) + trailer[5] = byte(l) + return +} + +func (sig *Signature) signPrepareHash(h hash.Hash) (digest []byte, err error) { + err = sig.buildHashSuffix() + if err != nil { + return + } + + h.Write(sig.HashSuffix) + digest = h.Sum(nil) + copy(sig.HashTag[:], digest) + return +} + +// Sign signs a message with a private key. The hash, h, must contain +// the hash of the message to be signed and will be mutated by this function. +// On success, the signature is stored in sig. Call Serialize to write it out. +// If config is nil, sensible defaults will be used. +func (sig *Signature) Sign(h hash.Hash, priv *PrivateKey, config *Config) (err error) { + sig.outSubpackets = sig.buildSubpackets() + digest, err := sig.signPrepareHash(h) + if err != nil { + return + } + + switch priv.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature.bytes, err = rsa.SignPKCS1v15(config.Random(), priv.PrivateKey.(*rsa.PrivateKey), sig.Hash, digest) + sig.RSASignature.bitLength = uint16(8 * len(sig.RSASignature.bytes)) + case PubKeyAlgoDSA: + dsaPriv := priv.PrivateKey.(*dsa.PrivateKey) + + // Need to truncate hashBytes to match FIPS 186-3 section 4.6. + subgroupSize := (dsaPriv.Q.BitLen() + 7) / 8 + if len(digest) > subgroupSize { + digest = digest[:subgroupSize] + } + r, s, err := dsa.Sign(config.Random(), dsaPriv, digest) + if err == nil { + sig.DSASigR.bytes = r.Bytes() + sig.DSASigR.bitLength = uint16(8 * len(sig.DSASigR.bytes)) + sig.DSASigS.bytes = s.Bytes() + sig.DSASigS.bitLength = uint16(8 * len(sig.DSASigS.bytes)) + } + default: + err = errors.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) + } + + return +} + +// SignUserId computes a signature from priv, asserting that pub is a valid +// key for the identity id. On success, the signature is stored in sig. Call +// Serialize to write it out. +// If config is nil, sensible defaults will be used. +func (sig *Signature) SignUserId(id string, pub *PublicKey, priv *PrivateKey, config *Config) error { + h, err := userIdSignatureHash(id, pub, sig.Hash) + if err != nil { + return nil + } + return sig.Sign(h, priv, config) +} + +// SignKey computes a signature from priv, asserting that pub is a subkey. On +// success, the signature is stored in sig. Call Serialize to write it out. +// If config is nil, sensible defaults will be used. +func (sig *Signature) SignKey(pub *PublicKey, priv *PrivateKey, config *Config) error { + h, err := keySignatureHash(&priv.PublicKey, pub, sig.Hash) + if err != nil { + return err + } + return sig.Sign(h, priv, config) +} + +// Serialize marshals sig to w. Sign, SignUserId or SignKey must have been +// called first. +func (sig *Signature) Serialize(w io.Writer) (err error) { + if len(sig.outSubpackets) == 0 { + sig.outSubpackets = sig.rawSubpackets + } + if sig.RSASignature.bytes == nil && sig.DSASigR.bytes == nil && sig.ECDSASigR.bytes == nil { + return errors.InvalidArgumentError("Signature: need to call Sign, SignUserId or SignKey before Serialize") + } + + sigLength := 0 + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sigLength = 2 + len(sig.RSASignature.bytes) + case PubKeyAlgoDSA: + sigLength = 2 + len(sig.DSASigR.bytes) + sigLength += 2 + len(sig.DSASigS.bytes) + case PubKeyAlgoECDSA: + sigLength = 2 + len(sig.ECDSASigR.bytes) + sigLength += 2 + len(sig.ECDSASigS.bytes) + default: + panic("impossible") + } + + unhashedSubpacketsLen := subpacketsLength(sig.outSubpackets, false) + length := len(sig.HashSuffix) - 6 /* trailer not included */ + + 2 /* length of unhashed subpackets */ + unhashedSubpacketsLen + + 2 /* hash tag */ + sigLength + err = serializeHeader(w, packetTypeSignature, length) + if err != nil { + return + } + + _, err = w.Write(sig.HashSuffix[:len(sig.HashSuffix)-6]) + if err != nil { + return + } + + unhashedSubpackets := make([]byte, 2+unhashedSubpacketsLen) + unhashedSubpackets[0] = byte(unhashedSubpacketsLen >> 8) + unhashedSubpackets[1] = byte(unhashedSubpacketsLen) + serializeSubpackets(unhashedSubpackets[2:], sig.outSubpackets, false) + + _, err = w.Write(unhashedSubpackets) + if err != nil { + return + } + _, err = w.Write(sig.HashTag[:]) + if err != nil { + return + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + err = writeMPIs(w, sig.RSASignature) + case PubKeyAlgoDSA: + err = writeMPIs(w, sig.DSASigR, sig.DSASigS) + case PubKeyAlgoECDSA: + err = writeMPIs(w, sig.ECDSASigR, sig.ECDSASigS) + default: + panic("impossible") + } + return +} + +// outputSubpacket represents a subpacket to be marshaled. +type outputSubpacket struct { + hashed bool // true if this subpacket is in the hashed area. + subpacketType signatureSubpacketType + isCritical bool + contents []byte +} + +func (sig *Signature) buildSubpackets() (subpackets []outputSubpacket) { + creationTime := make([]byte, 4) + binary.BigEndian.PutUint32(creationTime, uint32(sig.CreationTime.Unix())) + subpackets = append(subpackets, outputSubpacket{true, creationTimeSubpacket, false, creationTime}) + + if sig.IssuerKeyId != nil { + keyId := make([]byte, 8) + binary.BigEndian.PutUint64(keyId, *sig.IssuerKeyId) + subpackets = append(subpackets, outputSubpacket{true, issuerSubpacket, false, keyId}) + } + + if sig.SigLifetimeSecs != nil && *sig.SigLifetimeSecs != 0 { + sigLifetime := make([]byte, 4) + binary.BigEndian.PutUint32(sigLifetime, *sig.SigLifetimeSecs) + subpackets = append(subpackets, outputSubpacket{true, signatureExpirationSubpacket, true, sigLifetime}) + } + + // Key flags may only appear in self-signatures or certification signatures. + + if sig.FlagsValid { + var flags byte + if sig.FlagCertify { + flags |= KeyFlagCertify + } + if sig.FlagSign { + flags |= KeyFlagSign + } + if sig.FlagEncryptCommunications { + flags |= KeyFlagEncryptCommunications + } + if sig.FlagEncryptStorage { + flags |= KeyFlagEncryptStorage + } + subpackets = append(subpackets, outputSubpacket{true, keyFlagsSubpacket, false, []byte{flags}}) + } + + // The following subpackets may only appear in self-signatures + + if sig.KeyLifetimeSecs != nil && *sig.KeyLifetimeSecs != 0 { + keyLifetime := make([]byte, 4) + binary.BigEndian.PutUint32(keyLifetime, *sig.KeyLifetimeSecs) + subpackets = append(subpackets, outputSubpacket{true, keyExpirationSubpacket, true, keyLifetime}) + } + + if sig.IsPrimaryId != nil && *sig.IsPrimaryId { + subpackets = append(subpackets, outputSubpacket{true, primaryUserIdSubpacket, false, []byte{1}}) + } + + if len(sig.PreferredSymmetric) > 0 { + subpackets = append(subpackets, outputSubpacket{true, prefSymmetricAlgosSubpacket, false, sig.PreferredSymmetric}) + } + + if len(sig.PreferredHash) > 0 { + subpackets = append(subpackets, outputSubpacket{true, prefHashAlgosSubpacket, false, sig.PreferredHash}) + } + + if len(sig.PreferredCompression) > 0 { + subpackets = append(subpackets, outputSubpacket{true, prefCompressionSubpacket, false, sig.PreferredCompression}) + } + + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_test.go new file mode 100644 index 00000000000..c1bbde8b0c3 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_test.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto" + "encoding/hex" + "testing" +) + +func TestSignatureRead(t *testing.T) { + packet, err := Read(readerFromHex(signatureDataHex)) + if err != nil { + t.Error(err) + return + } + sig, ok := packet.(*Signature) + if !ok || sig.SigType != SigTypeBinary || sig.PubKeyAlgo != PubKeyAlgoRSA || sig.Hash != crypto.SHA1 { + t.Errorf("failed to parse, got: %#v", packet) + } +} + +func TestSignatureReserialize(t *testing.T) { + packet, _ := Read(readerFromHex(signatureDataHex)) + sig := packet.(*Signature) + out := new(bytes.Buffer) + err := sig.Serialize(out) + if err != nil { + t.Errorf("error reserializing: %s", err) + return + } + + expected, _ := hex.DecodeString(signatureDataHex) + if !bytes.Equal(expected, out.Bytes()) { + t.Errorf("output doesn't match input (got vs expected):\n%s\n%s", hex.Dump(out.Bytes()), hex.Dump(expected)) + } +} + +const signatureDataHex = "c2c05c04000102000605024cb45112000a0910ab105c91af38fb158f8d07ff5596ea368c5efe015bed6e78348c0f033c931d5f2ce5db54ce7f2a7e4b4ad64db758d65a7a71773edeab7ba2a9e0908e6a94a1175edd86c1d843279f045b021a6971a72702fcbd650efc393c5474d5b59a15f96d2eaad4c4c426797e0dcca2803ef41c6ff234d403eec38f31d610c344c06f2401c262f0993b2e66cad8a81ebc4322c723e0d4ba09fe917e8777658307ad8329adacba821420741009dfe87f007759f0982275d028a392c6ed983a0d846f890b36148c7358bdb8a516007fac760261ecd06076813831a36d0459075d1befa245ae7f7fb103d92ca759e9498fe60ef8078a39a3beda510deea251ea9f0a7f0df6ef42060f20780360686f3e400e" diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_v3.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_v3.go new file mode 100644 index 00000000000..6edff889349 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_v3.go @@ -0,0 +1,146 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "encoding/binary" + "fmt" + "io" + "strconv" + "time" + + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/s2k" +) + +// SignatureV3 represents older version 3 signatures. These signatures are less secure +// than version 4 and should not be used to create new signatures. They are included +// here for backwards compatibility to read and validate with older key material. +// See RFC 4880, section 5.2.2. +type SignatureV3 struct { + SigType SignatureType + CreationTime time.Time + IssuerKeyId uint64 + PubKeyAlgo PublicKeyAlgorithm + Hash crypto.Hash + HashTag [2]byte + + RSASignature parsedMPI + DSASigR, DSASigS parsedMPI +} + +func (sig *SignatureV3) parse(r io.Reader) (err error) { + // RFC 4880, section 5.2.2 + var buf [8]byte + if _, err = readFull(r, buf[:1]); err != nil { + return + } + if buf[0] < 2 || buf[0] > 3 { + err = errors.UnsupportedError("signature packet version " + strconv.Itoa(int(buf[0]))) + return + } + if _, err = readFull(r, buf[:1]); err != nil { + return + } + if buf[0] != 5 { + err = errors.UnsupportedError( + "invalid hashed material length " + strconv.Itoa(int(buf[0]))) + return + } + + // Read hashed material: signature type + creation time + if _, err = readFull(r, buf[:5]); err != nil { + return + } + sig.SigType = SignatureType(buf[0]) + t := binary.BigEndian.Uint32(buf[1:5]) + sig.CreationTime = time.Unix(int64(t), 0) + + // Eight-octet Key ID of signer. + if _, err = readFull(r, buf[:8]); err != nil { + return + } + sig.IssuerKeyId = binary.BigEndian.Uint64(buf[:]) + + // Public-key and hash algorithm + if _, err = readFull(r, buf[:2]); err != nil { + return + } + sig.PubKeyAlgo = PublicKeyAlgorithm(buf[0]) + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoDSA: + default: + err = errors.UnsupportedError("public key algorithm " + strconv.Itoa(int(sig.PubKeyAlgo))) + return + } + var ok bool + if sig.Hash, ok = s2k.HashIdToHash(buf[1]); !ok { + return errors.UnsupportedError("hash function " + strconv.Itoa(int(buf[2]))) + } + + // Two-octet field holding left 16 bits of signed hash value. + if _, err = readFull(r, sig.HashTag[:2]); err != nil { + return + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature.bytes, sig.RSASignature.bitLength, err = readMPI(r) + case PubKeyAlgoDSA: + if sig.DSASigR.bytes, sig.DSASigR.bitLength, err = readMPI(r); err != nil { + return + } + sig.DSASigS.bytes, sig.DSASigS.bitLength, err = readMPI(r) + default: + panic("unreachable") + } + return +} + +// Serialize marshals sig to w. Sign, SignUserId or SignKey must have been +// called first. +func (sig *SignatureV3) Serialize(w io.Writer) (err error) { + buf := make([]byte, 8) + + // Write the sig type and creation time + buf[0] = byte(sig.SigType) + binary.BigEndian.PutUint32(buf[1:5], uint32(sig.CreationTime.Unix())) + if _, err = w.Write(buf[:5]); err != nil { + return + } + + // Write the issuer long key ID + binary.BigEndian.PutUint64(buf[:8], sig.IssuerKeyId) + if _, err = w.Write(buf[:8]); err != nil { + return + } + + // Write public key algorithm, hash ID, and hash value + buf[0] = byte(sig.PubKeyAlgo) + hashId, ok := s2k.HashToHashId(sig.Hash) + if !ok { + return errors.UnsupportedError(fmt.Sprintf("hash function %v", sig.Hash)) + } + buf[1] = hashId + copy(buf[2:4], sig.HashTag[:]) + if _, err = w.Write(buf[:4]); err != nil { + return + } + + if sig.RSASignature.bytes == nil && sig.DSASigR.bytes == nil { + return errors.InvalidArgumentError("Signature: need to call Sign, SignUserId or SignKey before Serialize") + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + err = writeMPIs(w, sig.RSASignature) + case PubKeyAlgoDSA: + err = writeMPIs(w, sig.DSASigR, sig.DSASigS) + default: + panic("impossible") + } + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_v3_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_v3_test.go new file mode 100644 index 00000000000..ad7b62ac193 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/signature_v3_test.go @@ -0,0 +1,92 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto" + "encoding/hex" + "io" + "io/ioutil" + "testing" + + "golang.org/x/crypto/openpgp/armor" +) + +func TestSignatureV3Read(t *testing.T) { + r := v3KeyReader(t) + Read(r) // Skip public key + Read(r) // Skip uid + packet, err := Read(r) // Signature + if err != nil { + t.Error(err) + return + } + sig, ok := packet.(*SignatureV3) + if !ok || sig.SigType != SigTypeGenericCert || sig.PubKeyAlgo != PubKeyAlgoRSA || sig.Hash != crypto.MD5 { + t.Errorf("failed to parse, got: %#v", packet) + } +} + +func TestSignatureV3Reserialize(t *testing.T) { + r := v3KeyReader(t) + Read(r) // Skip public key + Read(r) // Skip uid + packet, err := Read(r) + if err != nil { + t.Error(err) + return + } + sig := packet.(*SignatureV3) + out := new(bytes.Buffer) + if err = sig.Serialize(out); err != nil { + t.Errorf("error reserializing: %s", err) + return + } + expected, err := ioutil.ReadAll(v3KeyReader(t)) + if err != nil { + t.Error(err) + return + } + expected = expected[4+141+4+39:] // See pgpdump offsets below, this is where the sig starts + if !bytes.Equal(expected, out.Bytes()) { + t.Errorf("output doesn't match input (got vs expected):\n%s\n%s", hex.Dump(out.Bytes()), hex.Dump(expected)) + } +} + +func v3KeyReader(t *testing.T) io.Reader { + armorBlock, err := armor.Decode(bytes.NewBufferString(keySigV3Armor)) + if err != nil { + t.Fatalf("armor Decode failed: %v", err) + } + return armorBlock.Body +} + +// keySigV3Armor is some V3 public key I found in an SKS dump. +// Old: Public Key Packet(tag 6)(141 bytes) +// Ver 4 - new +// Public key creation time - Fri Sep 16 17:13:54 CDT 1994 +// Pub alg - unknown(pub 0) +// Unknown public key(pub 0) +// Old: User ID Packet(tag 13)(39 bytes) +// User ID - Armin M. Warda +// Old: Signature Packet(tag 2)(149 bytes) +// Ver 4 - new +// Sig type - unknown(05) +// Pub alg - ElGamal Encrypt-Only(pub 16) +// Hash alg - unknown(hash 46) +// Hashed Sub: unknown(sub 81, critical)(1988 bytes) +const keySigV3Armor = `-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: SKS 1.0.10 + +mI0CLnoYogAAAQQA1qwA2SuJwfQ5bCQ6u5t20ulnOtY0gykf7YjiK4LiVeRBwHjGq7v30tGV +5Qti7qqRW4Ww7CDCJc4sZMFnystucR2vLkXaSoNWoFm4Fg47NiisDdhDezHwbVPW6OpCFNSi +ZAamtj4QAUBu8j4LswafrJqZqR9336/V3g8Yil2l48kABRG0J0FybWluIE0uIFdhcmRhIDx3 +YXJkYUBuZXBoaWxpbS5ydWhyLmRlPoiVAgUQLok2xwXR6zmeWEiZAQE/DgP/WgxPQh40/Po4 +gSkWZCDAjNdph7zexvAb0CcUWahcwiBIgg3U5ErCx9I5CNVA9U+s8bNrDZwgSIeBzp3KhWUx +524uhGgm6ZUTOAIKA6CbV6pfqoLpJnRYvXYQU5mIWsNa99wcu2qu18OeEDnztb7aLA6Ra9OF +YFCbq4EjXRoOrYM= +=LPjs +-----END PGP PUBLIC KEY BLOCK-----` diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetric_key_encrypted.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetric_key_encrypted.go new file mode 100644 index 00000000000..7dff6552212 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetric_key_encrypted.go @@ -0,0 +1,164 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/cipher" + "io" + "strconv" + + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/s2k" +) + +// This is the largest session key that we'll support. Since no 512-bit cipher +// has even been seriously used, this is comfortably large. +const maxSessionKeySizeInBytes = 64 + +// SymmetricKeyEncrypted represents a passphrase protected session key. See RFC +// 4880, section 5.3. +type SymmetricKeyEncrypted struct { + CipherFunc CipherFunction + Encrypted bool + Key []byte // Empty unless Encrypted is false. + s2k func(out, in []byte) + encryptedKey []byte +} + +const symmetricKeyEncryptedVersion = 4 + +func (ske *SymmetricKeyEncrypted) parse(r io.Reader) (err error) { + // RFC 4880, section 5.3. + var buf [2]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != symmetricKeyEncryptedVersion { + return errors.UnsupportedError("SymmetricKeyEncrypted version") + } + ske.CipherFunc = CipherFunction(buf[1]) + + if ske.CipherFunc.KeySize() == 0 { + return errors.UnsupportedError("unknown cipher: " + strconv.Itoa(int(buf[1]))) + } + + ske.s2k, err = s2k.Parse(r) + if err != nil { + return + } + + encryptedKey := make([]byte, maxSessionKeySizeInBytes) + // The session key may follow. We just have to try and read to find + // out. If it exists then we limit it to maxSessionKeySizeInBytes. + n, err := readFull(r, encryptedKey) + if err != nil && err != io.ErrUnexpectedEOF { + return + } + err = nil + if n != 0 { + if n == maxSessionKeySizeInBytes { + return errors.UnsupportedError("oversized encrypted session key") + } + ske.encryptedKey = encryptedKey[:n] + } + + ske.Encrypted = true + + return +} + +// Decrypt attempts to decrypt an encrypted session key. If it returns nil, +// ske.Key will contain the session key. +func (ske *SymmetricKeyEncrypted) Decrypt(passphrase []byte) error { + if !ske.Encrypted { + return nil + } + + key := make([]byte, ske.CipherFunc.KeySize()) + ske.s2k(key, passphrase) + + if len(ske.encryptedKey) == 0 { + ske.Key = key + } else { + // the IV is all zeros + iv := make([]byte, ske.CipherFunc.blockSize()) + c := cipher.NewCFBDecrypter(ske.CipherFunc.new(key), iv) + c.XORKeyStream(ske.encryptedKey, ske.encryptedKey) + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + if ske.CipherFunc.blockSize() == 0 { + return errors.UnsupportedError("unknown cipher: " + strconv.Itoa(int(ske.CipherFunc))) + } + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + ske.Key = ske.encryptedKey[1:] + if len(ske.Key)%ske.CipherFunc.blockSize() != 0 { + ske.Key = nil + return errors.StructuralError("length of decrypted key not a multiple of block size") + } + } + + ske.Encrypted = false + return nil +} + +// SerializeSymmetricKeyEncrypted serializes a symmetric key packet to w. The +// packet contains a random session key, encrypted by a key derived from the +// given passphrase. The session key is returned and must be passed to +// SerializeSymmetricallyEncrypted. +// If config is nil, sensible defaults will be used. +func SerializeSymmetricKeyEncrypted(w io.Writer, passphrase []byte, config *Config) (key []byte, err error) { + cipherFunc := config.Cipher() + keySize := cipherFunc.KeySize() + if keySize == 0 { + return nil, errors.UnsupportedError("unknown cipher: " + strconv.Itoa(int(cipherFunc))) + } + + s2kBuf := new(bytes.Buffer) + keyEncryptingKey := make([]byte, keySize) + // s2k.Serialize salts and stretches the passphrase, and writes the + // resulting key to keyEncryptingKey and the s2k descriptor to s2kBuf. + err = s2k.Serialize(s2kBuf, keyEncryptingKey, config.Random(), passphrase, &s2k.Config{Hash: config.Hash(), S2KCount: config.PasswordHashIterations()}) + if err != nil { + return + } + s2kBytes := s2kBuf.Bytes() + + packetLength := 2 /* header */ + len(s2kBytes) + 1 /* cipher type */ + keySize + err = serializeHeader(w, packetTypeSymmetricKeyEncrypted, packetLength) + if err != nil { + return + } + + var buf [2]byte + buf[0] = symmetricKeyEncryptedVersion + buf[1] = byte(cipherFunc) + _, err = w.Write(buf[:]) + if err != nil { + return + } + _, err = w.Write(s2kBytes) + if err != nil { + return + } + + sessionKey := make([]byte, keySize) + _, err = io.ReadFull(config.Random(), sessionKey) + if err != nil { + return + } + iv := make([]byte, cipherFunc.blockSize()) + c := cipher.NewCFBEncrypter(cipherFunc.new(keyEncryptingKey), iv) + encryptedCipherAndKey := make([]byte, keySize+1) + c.XORKeyStream(encryptedCipherAndKey, buf[1:]) + c.XORKeyStream(encryptedCipherAndKey[1:], sessionKey) + _, err = w.Write(encryptedCipherAndKey) + if err != nil { + return + } + + key = sessionKey + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetric_key_encrypted_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetric_key_encrypted_test.go new file mode 100644 index 00000000000..dd983cb387a --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetric_key_encrypted_test.go @@ -0,0 +1,102 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "io" + "io/ioutil" + "testing" +) + +func TestSymmetricKeyEncrypted(t *testing.T) { + buf := readerFromHex(symmetricallyEncryptedHex) + packet, err := Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricKeyEncrypted: %s", err) + return + } + ske, ok := packet.(*SymmetricKeyEncrypted) + if !ok { + t.Error("didn't find SymmetricKeyEncrypted packet") + return + } + err = ske.Decrypt([]byte("password")) + if err != nil { + t.Error(err) + return + } + + packet, err = Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricallyEncrypted: %s", err) + return + } + se, ok := packet.(*SymmetricallyEncrypted) + if !ok { + t.Error("didn't find SymmetricallyEncrypted packet") + return + } + r, err := se.Decrypt(ske.CipherFunc, ske.Key) + if err != nil { + t.Error(err) + return + } + + contents, err := ioutil.ReadAll(r) + if err != nil && err != io.EOF { + t.Error(err) + return + } + + expectedContents, _ := hex.DecodeString(symmetricallyEncryptedContentsHex) + if !bytes.Equal(expectedContents, contents) { + t.Errorf("bad contents got:%x want:%x", contents, expectedContents) + } +} + +const symmetricallyEncryptedHex = "8c0d04030302371a0b38d884f02060c91cf97c9973b8e58e028e9501708ccfe618fb92afef7fa2d80ddadd93cf" +const symmetricallyEncryptedContentsHex = "cb1062004d14c4df636f6e74656e74732e0a" + +func TestSerializeSymmetricKeyEncrypted(t *testing.T) { + buf := bytes.NewBuffer(nil) + passphrase := []byte("testing") + config := &Config{ + DefaultCipher: CipherAES128, + } + + key, err := SerializeSymmetricKeyEncrypted(buf, passphrase, config) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + p, err := Read(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + ske, ok := p.(*SymmetricKeyEncrypted) + if !ok { + t.Errorf("parsed a different packet type: %#v", p) + return + } + + if !ske.Encrypted { + t.Errorf("SKE not encrypted but should be") + } + if ske.CipherFunc != config.DefaultCipher { + t.Errorf("SKE cipher function is %d (expected %d)", ske.CipherFunc, config.DefaultCipher) + } + err = ske.Decrypt(passphrase) + if err != nil { + t.Errorf("failed to decrypt reparsed SKE: %s", err) + return + } + if !bytes.Equal(key, ske.Key) { + t.Errorf("keys don't match after Decrpyt: %x (original) vs %x (parsed)", key, ske.Key) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetrically_encrypted.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetrically_encrypted.go new file mode 100644 index 00000000000..6126030eb90 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetrically_encrypted.go @@ -0,0 +1,290 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/cipher" + "crypto/sha1" + "crypto/subtle" + "golang.org/x/crypto/openpgp/errors" + "hash" + "io" + "strconv" +) + +// SymmetricallyEncrypted represents a symmetrically encrypted byte string. The +// encrypted contents will consist of more OpenPGP packets. See RFC 4880, +// sections 5.7 and 5.13. +type SymmetricallyEncrypted struct { + MDC bool // true iff this is a type 18 packet and thus has an embedded MAC. + contents io.Reader + prefix []byte +} + +const symmetricallyEncryptedVersion = 1 + +func (se *SymmetricallyEncrypted) parse(r io.Reader) error { + if se.MDC { + // See RFC 4880, section 5.13. + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + if buf[0] != symmetricallyEncryptedVersion { + return errors.UnsupportedError("unknown SymmetricallyEncrypted version") + } + } + se.contents = r + return nil +} + +// Decrypt returns a ReadCloser, from which the decrypted contents of the +// packet can be read. An incorrect key can, with high probability, be detected +// immediately and this will result in a KeyIncorrect error being returned. +func (se *SymmetricallyEncrypted) Decrypt(c CipherFunction, key []byte) (io.ReadCloser, error) { + keySize := c.KeySize() + if keySize == 0 { + return nil, errors.UnsupportedError("unknown cipher: " + strconv.Itoa(int(c))) + } + if len(key) != keySize { + return nil, errors.InvalidArgumentError("SymmetricallyEncrypted: incorrect key length") + } + + if se.prefix == nil { + se.prefix = make([]byte, c.blockSize()+2) + _, err := readFull(se.contents, se.prefix) + if err != nil { + return nil, err + } + } else if len(se.prefix) != c.blockSize()+2 { + return nil, errors.InvalidArgumentError("can't try ciphers with different block lengths") + } + + ocfbResync := OCFBResync + if se.MDC { + // MDC packets use a different form of OCFB mode. + ocfbResync = OCFBNoResync + } + + s := NewOCFBDecrypter(c.new(key), se.prefix, ocfbResync) + if s == nil { + return nil, errors.ErrKeyIncorrect + } + + plaintext := cipher.StreamReader{S: s, R: se.contents} + + if se.MDC { + // MDC packets have an embedded hash that we need to check. + h := sha1.New() + h.Write(se.prefix) + return &seMDCReader{in: plaintext, h: h}, nil + } + + // Otherwise, we just need to wrap plaintext so that it's a valid ReadCloser. + return seReader{plaintext}, nil +} + +// seReader wraps an io.Reader with a no-op Close method. +type seReader struct { + in io.Reader +} + +func (ser seReader) Read(buf []byte) (int, error) { + return ser.in.Read(buf) +} + +func (ser seReader) Close() error { + return nil +} + +const mdcTrailerSize = 1 /* tag byte */ + 1 /* length byte */ + sha1.Size + +// An seMDCReader wraps an io.Reader, maintains a running hash and keeps hold +// of the most recent 22 bytes (mdcTrailerSize). Upon EOF, those bytes form an +// MDC packet containing a hash of the previous contents which is checked +// against the running hash. See RFC 4880, section 5.13. +type seMDCReader struct { + in io.Reader + h hash.Hash + trailer [mdcTrailerSize]byte + scratch [mdcTrailerSize]byte + trailerUsed int + error bool + eof bool +} + +func (ser *seMDCReader) Read(buf []byte) (n int, err error) { + if ser.error { + err = io.ErrUnexpectedEOF + return + } + if ser.eof { + err = io.EOF + return + } + + // If we haven't yet filled the trailer buffer then we must do that + // first. + for ser.trailerUsed < mdcTrailerSize { + n, err = ser.in.Read(ser.trailer[ser.trailerUsed:]) + ser.trailerUsed += n + if err == io.EOF { + if ser.trailerUsed != mdcTrailerSize { + n = 0 + err = io.ErrUnexpectedEOF + ser.error = true + return + } + ser.eof = true + n = 0 + return + } + + if err != nil { + n = 0 + return + } + } + + // If it's a short read then we read into a temporary buffer and shift + // the data into the caller's buffer. + if len(buf) <= mdcTrailerSize { + n, err = readFull(ser.in, ser.scratch[:len(buf)]) + copy(buf, ser.trailer[:n]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], ser.trailer[n:]) + copy(ser.trailer[mdcTrailerSize-n:], ser.scratch[:]) + if n < len(buf) { + ser.eof = true + err = io.EOF + } + return + } + + n, err = ser.in.Read(buf[mdcTrailerSize:]) + copy(buf, ser.trailer[:]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], buf[n:]) + + if err == io.EOF { + ser.eof = true + } + return +} + +// This is a new-format packet tag byte for a type 19 (MDC) packet. +const mdcPacketTagByte = byte(0x80) | 0x40 | 19 + +func (ser *seMDCReader) Close() error { + if ser.error { + return errors.SignatureError("error during reading") + } + + for !ser.eof { + // We haven't seen EOF so we need to read to the end + var buf [1024]byte + _, err := ser.Read(buf[:]) + if err == io.EOF { + break + } + if err != nil { + return errors.SignatureError("error during reading") + } + } + + if ser.trailer[0] != mdcPacketTagByte || ser.trailer[1] != sha1.Size { + return errors.SignatureError("MDC packet not found") + } + ser.h.Write(ser.trailer[:2]) + + final := ser.h.Sum(nil) + if subtle.ConstantTimeCompare(final, ser.trailer[2:]) != 1 { + return errors.SignatureError("hash mismatch") + } + return nil +} + +// An seMDCWriter writes through to an io.WriteCloser while maintains a running +// hash of the data written. On close, it emits an MDC packet containing the +// running hash. +type seMDCWriter struct { + w io.WriteCloser + h hash.Hash +} + +func (w *seMDCWriter) Write(buf []byte) (n int, err error) { + w.h.Write(buf) + return w.w.Write(buf) +} + +func (w *seMDCWriter) Close() (err error) { + var buf [mdcTrailerSize]byte + + buf[0] = mdcPacketTagByte + buf[1] = sha1.Size + w.h.Write(buf[:2]) + digest := w.h.Sum(nil) + copy(buf[2:], digest) + + _, err = w.w.Write(buf[:]) + if err != nil { + return + } + return w.w.Close() +} + +// noOpCloser is like an ioutil.NopCloser, but for an io.Writer. +type noOpCloser struct { + w io.Writer +} + +func (c noOpCloser) Write(data []byte) (n int, err error) { + return c.w.Write(data) +} + +func (c noOpCloser) Close() error { + return nil +} + +// SerializeSymmetricallyEncrypted serializes a symmetrically encrypted packet +// to w and returns a WriteCloser to which the to-be-encrypted packets can be +// written. +// If config is nil, sensible defaults will be used. +func SerializeSymmetricallyEncrypted(w io.Writer, c CipherFunction, key []byte, config *Config) (contents io.WriteCloser, err error) { + if c.KeySize() != len(key) { + return nil, errors.InvalidArgumentError("SymmetricallyEncrypted.Serialize: bad key length") + } + writeCloser := noOpCloser{w} + ciphertext, err := serializeStreamHeader(writeCloser, packetTypeSymmetricallyEncryptedMDC) + if err != nil { + return + } + + _, err = ciphertext.Write([]byte{symmetricallyEncryptedVersion}) + if err != nil { + return + } + + block := c.new(key) + blockSize := block.BlockSize() + iv := make([]byte, blockSize) + _, err = config.Random().Read(iv) + if err != nil { + return + } + s, prefix := NewOCFBEncrypter(block, iv, OCFBNoResync) + _, err = ciphertext.Write(prefix) + if err != nil { + return + } + plaintext := cipher.StreamWriter{S: s, W: ciphertext} + + h := sha1.New() + h.Write(iv) + h.Write(iv[blockSize-2:]) + contents = &seMDCWriter{w: plaintext, h: h} + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetrically_encrypted_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetrically_encrypted_test.go new file mode 100644 index 00000000000..c5c00f7b9c3 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/symmetrically_encrypted_test.go @@ -0,0 +1,123 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "golang.org/x/crypto/openpgp/errors" + "io" + "io/ioutil" + "testing" +) + +// TestReader wraps a []byte and returns reads of a specific length. +type testReader struct { + data []byte + stride int +} + +func (t *testReader) Read(buf []byte) (n int, err error) { + n = t.stride + if n > len(t.data) { + n = len(t.data) + } + if n > len(buf) { + n = len(buf) + } + copy(buf, t.data) + t.data = t.data[n:] + if len(t.data) == 0 { + err = io.EOF + } + return +} + +func testMDCReader(t *testing.T) { + mdcPlaintext, _ := hex.DecodeString(mdcPlaintextHex) + + for stride := 1; stride < len(mdcPlaintext)/2; stride++ { + r := &testReader{data: mdcPlaintext, stride: stride} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + body, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("stride: %d, error: %s", stride, err) + continue + } + if !bytes.Equal(body, mdcPlaintext[:len(mdcPlaintext)-22]) { + t.Errorf("stride: %d: bad contents %x", stride, body) + continue + } + + err = mdcReader.Close() + if err != nil { + t.Errorf("stride: %d, error on Close: %s", stride, err) + } + } + + mdcPlaintext[15] ^= 80 + + r := &testReader{data: mdcPlaintext, stride: 2} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + _, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("corruption test, error: %s", err) + return + } + err = mdcReader.Close() + if err == nil { + t.Error("corruption: no error") + } else if _, ok := err.(*errors.SignatureError); !ok { + t.Errorf("corruption: expected SignatureError, got: %s", err) + } +} + +const mdcPlaintextHex = "a302789c3b2d93c4e0eb9aba22283539b3203335af44a134afb800c849cb4c4de10200aff40b45d31432c80cb384299a0655966d6939dfdeed1dddf980" + +func TestSerialize(t *testing.T) { + buf := bytes.NewBuffer(nil) + c := CipherAES128 + key := make([]byte, c.KeySize()) + + w, err := SerializeSymmetricallyEncrypted(buf, c, key, nil) + if err != nil { + t.Errorf("error from SerializeSymmetricallyEncrypted: %s", err) + return + } + + contents := []byte("hello world\n") + + w.Write(contents) + w.Close() + + p, err := Read(buf) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + se, ok := p.(*SymmetricallyEncrypted) + if !ok { + t.Errorf("didn't read a *SymmetricallyEncrypted") + return + } + + r, err := se.Decrypt(c, key) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + contentsCopy := bytes.NewBuffer(nil) + _, err = io.Copy(contentsCopy, r) + if err != nil { + t.Errorf("error from io.Copy: %s", err) + return + } + if !bytes.Equal(contentsCopy.Bytes(), contents) { + t.Errorf("contents not equal got: %x want: %x", contentsCopy.Bytes(), contents) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userattribute.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userattribute.go new file mode 100644 index 00000000000..96a2b382a1d --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userattribute.go @@ -0,0 +1,91 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "image" + "image/jpeg" + "io" + "io/ioutil" +) + +const UserAttrImageSubpacket = 1 + +// UserAttribute is capable of storing other types of data about a user +// beyond name, email and a text comment. In practice, user attributes are typically used +// to store a signed thumbnail photo JPEG image of the user. +// See RFC 4880, section 5.12. +type UserAttribute struct { + Contents []*OpaqueSubpacket +} + +// NewUserAttributePhoto creates a user attribute packet +// containing the given images. +func NewUserAttributePhoto(photos ...image.Image) (uat *UserAttribute, err error) { + uat = new(UserAttribute) + for _, photo := range photos { + var buf bytes.Buffer + // RFC 4880, Section 5.12.1. + data := []byte{ + 0x10, 0x00, // Little-endian image header length (16 bytes) + 0x01, // Image header version 1 + 0x01, // JPEG + 0, 0, 0, 0, // 12 reserved octets, must be all zero. + 0, 0, 0, 0, + 0, 0, 0, 0} + if _, err = buf.Write(data); err != nil { + return + } + if err = jpeg.Encode(&buf, photo, nil); err != nil { + return + } + uat.Contents = append(uat.Contents, &OpaqueSubpacket{ + SubType: UserAttrImageSubpacket, + Contents: buf.Bytes()}) + } + return +} + +// NewUserAttribute creates a new user attribute packet containing the given subpackets. +func NewUserAttribute(contents ...*OpaqueSubpacket) *UserAttribute { + return &UserAttribute{Contents: contents} +} + +func (uat *UserAttribute) parse(r io.Reader) (err error) { + // RFC 4880, section 5.13 + b, err := ioutil.ReadAll(r) + if err != nil { + return + } + uat.Contents, err = OpaqueSubpackets(b) + return +} + +// Serialize marshals the user attribute to w in the form of an OpenPGP packet, including +// header. +func (uat *UserAttribute) Serialize(w io.Writer) (err error) { + var buf bytes.Buffer + for _, sp := range uat.Contents { + sp.Serialize(&buf) + } + if err = serializeHeader(w, packetTypeUserAttribute, buf.Len()); err != nil { + return err + } + _, err = w.Write(buf.Bytes()) + return +} + +// ImageData returns zero or more byte slices, each containing +// JPEG File Interchange Format (JFIF), for each photo in the +// the user attribute packet. +func (uat *UserAttribute) ImageData() (imageData [][]byte) { + for _, sp := range uat.Contents { + if sp.SubType == UserAttrImageSubpacket && len(sp.Contents) > 16 { + imageData = append(imageData, sp.Contents[16:]) + } + } + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userattribute_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userattribute_test.go new file mode 100644 index 00000000000..13ca5143cee --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userattribute_test.go @@ -0,0 +1,109 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/base64" + "image/color" + "image/jpeg" + "testing" +) + +func TestParseUserAttribute(t *testing.T) { + r := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(userAttributePacket)) + for i := 0; i < 2; i++ { + p, err := Read(r) + if err != nil { + t.Fatal(err) + } + uat := p.(*UserAttribute) + imgs := uat.ImageData() + if len(imgs) != 1 { + t.Errorf("Unexpected number of images in user attribute packet: %d", len(imgs)) + } + if len(imgs[0]) != 3395 { + t.Errorf("Unexpected JPEG image size: %d", len(imgs[0])) + } + img, err := jpeg.Decode(bytes.NewBuffer(imgs[0])) + if err != nil { + t.Errorf("Error decoding JPEG image: %v", err) + } + // A pixel in my right eye. + pixel := color.NRGBAModel.Convert(img.At(56, 36)) + ref := color.NRGBA{R: 157, G: 128, B: 124, A: 255} + if pixel != ref { + t.Errorf("Unexpected pixel color: %v", pixel) + } + w := bytes.NewBuffer(nil) + err = uat.Serialize(w) + if err != nil { + t.Errorf("Error writing user attribute: %v", err) + } + r = bytes.NewBuffer(w.Bytes()) + } +} + +const userAttributePacket = ` +0cyWzJQBEAABAQAAAAAAAAAAAAAAAP/Y/+AAEEpGSUYAAQIAAAEAAQAA/9sAQwAFAwQEBAMFBAQE +BQUFBgcMCAcHBwcPCgsJDBEPEhIRDxEQExYcFxMUGhUQERghGBocHR8fHxMXIiQiHiQcHh8e/9sA +QwEFBQUHBgcOCAgOHhQRFB4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4e/8AAEQgAZABkAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYH +CAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHw +JDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6 +g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk +5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIB +AgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEX +GBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKT +lJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX2 +9/j5+v/aAAwDAQACEQMRAD8A5uGP06VehQ4pIox04q5EnHSvAep+hIIl4zVuMHGPWmRrUWtalaaN +pU2oXsgSGJSxPr6ClvoitErs0Itqjc7BQOpPAFYmrfEnwjojtHNqaXEynBjtx5hH4jj9a8B8d+Od +W8UXZjWR4LJT+7t0Jwfc+prnIdO1CWZEW2mZ3HyDactXXDB3V5s8evm1namj6r0H4weCLtxG+ova +ueP30RA/MV6not1bX0Ed1ZzxzwyDKvGwZSPqK+Ff+ES8R8t/ZV2oHUmM10Hgbxp4m8BatEfNnWBH +/eWshOxx9Kmpg4te49RUM1kn+8Wh9zQ4P1FaMC7l465rjPh14y0fxnoseoaXOpfaPOgJ+eI98j09 +67W19M15bi4uzPSqTU480WXkjZkAyAR61DPE6OCSOalWRRgZxjvTb598sfU4FBwx5uY4T4feIm8P +TeJbAgc65NIM+8cX+FFeLfF3Vr3SfiNrMFrMypJMJcDPUqP8KK+kpVFyLU+ar037SXqX4hxVpMY7 +1UhPpVlT2rybKx9smWYz3NeH/EDVLzxt40j8O6bITaQybPlbKkjq39K9O8fasdH8IahfKxWQRFIy +Ou9uB/OuE/Z/0y3j1d9TuyoZCMs5xjuea1pLli5nn46q240l13PcfhN8EvDNtpcEl/CklyVBLuMk +mvU/Dfwo0BL/AO13FjEDD/qyV7Vn+CvGPg8zRpJrVm8ikLtEg6+1ew2dxZ3EQaJgysuQPasH7eXW +1zzsbVhT92kk/PsYieEND+zlPs6c/wCyAPyryH4wfCPRtW0u6j+xRLOxLxSoADkDpXY+MPjJ4c0S +9k082d3O8ZKkxw5XI96ytK+IGk+IpFjRpod+Qq3C7QT6A1E6NenaXbqRg6rlLlqS0fRnxjpd1r/w +w8afa7GWRPKbZLGeBKmeVNfZngLxNaeKfDdprVjxHcLlkJ5Vh1H5185/tDad9h8XOsqAw3Cb0cjq +CfX61P8AsveKf7L8T3fhe5nxa3g324YniQdh9R/KuivTdSmp9TXB1/Z1nRlsfU249QBx1pWfcwI7 +Cq6u2Ovamb9rYz16V5x7Psz5q/aJhZfibcupIElvE3H+7j+lFbXx9szP45jlUfeso8/99OKK9elL +3EeNVopzZVharCtxVRGGMk02S5JyFOB69zWTieypnL/GksfB+0cr9oQt69awPhPpD69Y3Ky3DWth +CWluGU4LAdq3vibGs/g68BJygVxjrwRW5+ztoRv/AAs8EeCZnO/J/hzz/Kumi4wp3kePjlOdZKPY +ml8Mvo6WM9ppi7J0EkQYMzkb1X0wW+bJHGACa+ivg14huZPCkjXUO6SImIYOQAP6UQ2sGneHmiWF +CYoSAAuM8etXfhBpMr+EZ3SSNRcMx6ZxWdes6ytBGSwkMNFuo7pnP614Ut9Zn1C4uLySKcwObGFA +Qnm4+XcR71h+CfDHiKCQWuv2YWFtw+bBZQD8rcE8n2Ney+GbGGQSM6I7xvtI681rXdp8hKRRp6t3 +FYPE1VDlsY1nQjWdl+J8w/tOeDZZ/AMd/EGefTHyxxyYjwfyODXg3waRh8UtEcFh+8Jb8FNfZPxh +Ak8J6nbPIsiyW7LnseK+Ofh99ptPHFnf2lu0y2twGcKuSEPB/Q1WHk50miq1o14TXU+xop+On61H +NMC6Nis1LgsAcUTSt1APFcXJZn0EqmhyvxA037friTYziBV6f7Tf40Vr3k4aXLx5OMZIzRXZB2ik +efJXbPHJJcnaD9aN2R1qoGO8/WkuLlIV+YjdjpXSonQ5lTxfiTwzqCnkeQxx9BWx+zPrQsrBFYja +zEfrXL6lfie3khcjY6lSPUGud+G3iA6FrY0uQ/KJsA9gCa0jSvFpnBi6tpKSPu++nsIfDFxeXciR +qIicscY4rxTwB8RUkn1axsPEf2LTYx85kTGzqCUP8VcJ47+JOs+I0Hhq1njjt/ufIeSvq1VtE+Gs +eoaUbSHUrkHdu3WtuX5Ix81XRh7OL5jirVpV5Whdn0F8C/iX4auVn0i612T7bASoe8wjTAd89K9g +vtSt5NMa4t5lkRhgOh3Dn6V8aaz8KZrIR3OlQ6r56LySmSxxz06Vo/CHx34h0rxBP4XvJ5AjK2RP +nEbAEj6ZxjPrWM6fMmoswqJxqJ1VZnqHxn1NLPwveqWHmNC2BnnNcD8DfDkGi+CH1m+ijN1qMzNA +4GSIiAMf+hVxPxU8Tapc3c0F9MGCn5GU5BX0Pau3+HmrT3XgXSIJCBHDGdgAx1NYSpezha52Yauq +1dya2Wh2onAIwTj1p0lxxWWLkhRyCKWa5O3ORXOos9KVQluZm83j0oqi84JyWH50Vdmc7ep43d3I +t1Z2Iz2FYdxeSTsxyRnvTdVuDNcNluM9KrKcg817NOnZGNbEXdkNckjrXGeIIprPxFFdRHAlIwem +COtdmxrG8Q2cd/ZNExw45RvQ1bVjim+dWNzw7eaTD4mN3dndCQCo6hmI5zXpj/Ea/wBHjkh0kwRW +xXEfl4yTxXzXZalJDL9nuWKMmRnHcV2Hh3WreCyYXW2SWQhd5P3F6n+lS43d2cTm6d7Ox9EWPxH1 +ODQxPqWpCaSU/ukUc4z3/WvKW8UhviAdaMewYZG98gj9c1ymoa8LyWOJHwkTDaVPb0qpr+q2m6Nb +cfvNo349az9mou9iZVXNWbub3jm98/Vza2ReV7lsJg/e3dsV654UR9N0K0sZP9ZDGFbHr3rzL4P+ +H7rXfEEWr3I3W1qf3IYdW9fwqDxf4k8UeH/G95p08kscHmk25dPlZT0we9YTj7SXKjpw1aNG8mj3 +FLv5ccU959ycnmvKPDnxB82YQarGsZPAlTp+IrvIr1ZIgySKwIyCOhFYTpyg9T0qWIhVV4svzPvf +IdhgY4orPachj81FRdmtzxqdiZmJ9aQEgdqZcPtmbJ71DJcAZ5r20kkeXJtsfPIQDwPzrG1a+S3i +LyHAHvmp7y7HOD1rlNdm+1T7Acovf3o+J2RMpezjzMvrob67pX9o2ShZlYgg/wAWKxZLLWLZ/Ke3 +mVh14yK9M+BMC3dre2ko3LHKCB7EV7EngeGQJdQ7HyBkMKS0djgq1W3c+XtK03U522RwzsTwNiEk +ntXoHgf4calql9El/G8UZbLfLyfr7V9FeGvh+s+0Lbxxcglu2K1NW1nwN4Gk/wBLuI57tV5jjwzE +/QVNS+0dWYRqNvXRFv4eeCodKsY1ggVIY1G3K4z714h+1Jqul3GpwaXYeXJLbzgyyrg4b+6D+HNb +vjz436zq9m+naHF/ZdkeGfOZXH17V4Vqt2b29K+ZuOc5bnce5zWdPBShL2lTfojSeJhy+zp/NjVz +1Bwa6DSfFGq6fbJFDKrov8DjPFcu97ZxsUe4jVhwVJ5Bpp1mwQiLewJPXacVq6fNpYyjOUXdHoKf +EG8VQHsInbuVcgflRXnt5fIs2FYHgcgUVi8LG+xusdW/mN7U2KgEVkTzPt60UVfQ9eHxGHrV1MGi +iD4V25x1qvdgLAMd6KK0pbHm4x++dp8FtUubLxJ5EIjMc+A4Za+qfD8pe1JZVOBmiinW3RyRPMfi +R8QPE638+k2l6LK0Hylbddhb6nOa80mlkcmWR2kcnlnOSaKK7qCXKcNdu5narcSrAoBxvODWJIga +VckjDdqKKwq/EaQ0gUdbjQ6mr7QGBUcd6tPBC6gtGpOOuKKKie5qn7qIpEXd0HSiiimSf//Z` diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userid.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userid.go new file mode 100644 index 00000000000..d6bea7d4acc --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userid.go @@ -0,0 +1,160 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "io" + "io/ioutil" + "strings" +) + +// UserId contains text that is intended to represent the name and email +// address of the key holder. See RFC 4880, section 5.11. By convention, this +// takes the form "Full Name (Comment) " +type UserId struct { + Id string // By convention, this takes the form "Full Name (Comment) " which is split out in the fields below. + + Name, Comment, Email string +} + +func hasInvalidCharacters(s string) bool { + for _, c := range s { + switch c { + case '(', ')', '<', '>', 0: + return true + } + } + return false +} + +// NewUserId returns a UserId or nil if any of the arguments contain invalid +// characters. The invalid characters are '\x00', '(', ')', '<' and '>' +func NewUserId(name, comment, email string) *UserId { + // RFC 4880 doesn't deal with the structure of userid strings; the + // name, comment and email form is just a convention. However, there's + // no convention about escaping the metacharacters and GPG just refuses + // to create user ids where, say, the name contains a '('. We mirror + // this behaviour. + + if hasInvalidCharacters(name) || hasInvalidCharacters(comment) || hasInvalidCharacters(email) { + return nil + } + + uid := new(UserId) + uid.Name, uid.Comment, uid.Email = name, comment, email + uid.Id = name + if len(comment) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "(" + uid.Id += comment + uid.Id += ")" + } + if len(email) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "<" + uid.Id += email + uid.Id += ">" + } + return uid +} + +func (uid *UserId) parse(r io.Reader) (err error) { + // RFC 4880, section 5.11 + b, err := ioutil.ReadAll(r) + if err != nil { + return + } + uid.Id = string(b) + uid.Name, uid.Comment, uid.Email = parseUserId(uid.Id) + return +} + +// Serialize marshals uid to w in the form of an OpenPGP packet, including +// header. +func (uid *UserId) Serialize(w io.Writer) error { + err := serializeHeader(w, packetTypeUserId, len(uid.Id)) + if err != nil { + return err + } + _, err = w.Write([]byte(uid.Id)) + return err +} + +// parseUserId extracts the name, comment and email from a user id string that +// is formatted as "Full Name (Comment) ". +func parseUserId(id string) (name, comment, email string) { + var n, c, e struct { + start, end int + } + var state int + + for offset, rune := range id { + switch state { + case 0: + // Entering name + n.start = offset + state = 1 + fallthrough + case 1: + // In name + if rune == '(' { + state = 2 + n.end = offset + } else if rune == '<' { + state = 5 + n.end = offset + } + case 2: + // Entering comment + c.start = offset + state = 3 + fallthrough + case 3: + // In comment + if rune == ')' { + state = 4 + c.end = offset + } + case 4: + // Between comment and email + if rune == '<' { + state = 5 + } + case 5: + // Entering email + e.start = offset + state = 6 + fallthrough + case 6: + // In email + if rune == '>' { + state = 7 + e.end = offset + } + default: + // After email + } + } + switch state { + case 1: + // ended in the name + n.end = len(id) + case 3: + // ended in comment + c.end = len(id) + case 6: + // ended in email + e.end = len(id) + } + + name = strings.TrimSpace(id[n.start:n.end]) + comment = strings.TrimSpace(id[c.start:c.end]) + email = strings.TrimSpace(id[e.start:e.end]) + return +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userid_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userid_test.go new file mode 100644 index 00000000000..29681938938 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/packet/userid_test.go @@ -0,0 +1,87 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" +) + +var userIdTests = []struct { + id string + name, comment, email string +}{ + {"", "", "", ""}, + {"John Smith", "John Smith", "", ""}, + {"John Smith ()", "John Smith", "", ""}, + {"John Smith () <>", "John Smith", "", ""}, + {"(comment", "", "comment", ""}, + {"(comment)", "", "comment", ""}, + {" sdfk", "", "", "email"}, + {" John Smith ( Comment ) asdkflj < email > lksdfj", "John Smith", "Comment", "email"}, + {" John Smith < email > lksdfj", "John Smith", "", "email"}, + {"("}, + {"foo", "bar", "", "foo (bar)"}, + {"foo", "", "baz", "foo "}, + {"", "bar", "baz", "(bar) "}, + {"foo", "bar", "baz", "foo (bar) "}, +} + +func TestNewUserId(t *testing.T) { + for i, test := range newUserIdTests { + uid := NewUserId(test.name, test.comment, test.email) + if uid == nil { + t.Errorf("#%d: returned nil", i) + continue + } + if uid.Id != test.id { + t.Errorf("#%d: got '%s', want '%s'", i, uid.Id, test.id) + } + } +} + +var invalidNewUserIdTests = []struct { + name, comment, email string +}{ + {"foo(", "", ""}, + {"foo<", "", ""}, + {"", "bar)", ""}, + {"", "bar<", ""}, + {"", "", "baz>"}, + {"", "", "baz)"}, + {"", "", "baz\x00"}, +} + +func TestNewUserIdWithInvalidInput(t *testing.T) { + for i, test := range invalidNewUserIdTests { + if uid := NewUserId(test.name, test.comment, test.email); uid != nil { + t.Errorf("#%d: returned non-nil value: %#v", i, uid) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/read.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/read.go new file mode 100644 index 00000000000..08129d70713 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/read.go @@ -0,0 +1,423 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package openpgp implements high level operations on OpenPGP messages. +package openpgp + +import ( + "crypto" + _ "crypto/sha256" + "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/packet" + "hash" + "io" + "strconv" +) + +// SignatureType is the armor type for a PGP signature. +var SignatureType = "PGP SIGNATURE" + +// readArmored reads an armored block with the given type. +func readArmored(r io.Reader, expectedType string) (body io.Reader, err error) { + block, err := armor.Decode(r) + if err != nil { + return + } + + if block.Type != expectedType { + return nil, errors.InvalidArgumentError("expected '" + expectedType + "', got: " + block.Type) + } + + return block.Body, nil +} + +// MessageDetails contains the result of parsing an OpenPGP encrypted and/or +// signed message. +type MessageDetails struct { + IsEncrypted bool // true if the message was encrypted. + EncryptedToKeyIds []uint64 // the list of recipient key ids. + IsSymmetricallyEncrypted bool // true if a passphrase could have decrypted the message. + DecryptedWith Key // the private key used to decrypt the message, if any. + IsSigned bool // true if the message is signed. + SignedByKeyId uint64 // the key id of the signer, if any. + SignedBy *Key // the key of the signer, if available. + LiteralData *packet.LiteralData // the metadata of the contents + UnverifiedBody io.Reader // the contents of the message. + + // If IsSigned is true and SignedBy is non-zero then the signature will + // be verified as UnverifiedBody is read. The signature cannot be + // checked until the whole of UnverifiedBody is read so UnverifiedBody + // must be consumed until EOF before the data can trusted. Even if a + // message isn't signed (or the signer is unknown) the data may contain + // an authentication code that is only checked once UnverifiedBody has + // been consumed. Once EOF has been seen, the following fields are + // valid. (An authentication code failure is reported as a + // SignatureError error when reading from UnverifiedBody.) + SignatureError error // nil if the signature is good. + Signature *packet.Signature // the signature packet itself. + + decrypted io.ReadCloser +} + +// A PromptFunction is used as a callback by functions that may need to decrypt +// a private key, or prompt for a passphrase. It is called with a list of +// acceptable, encrypted private keys and a boolean that indicates whether a +// passphrase is usable. It should either decrypt a private key or return a +// passphrase to try. If the decrypted private key or given passphrase isn't +// correct, the function will be called again, forever. Any error returned will +// be passed up. +type PromptFunction func(keys []Key, symmetric bool) ([]byte, error) + +// A keyEnvelopePair is used to store a private key with the envelope that +// contains a symmetric key, encrypted with that key. +type keyEnvelopePair struct { + key Key + encryptedKey *packet.EncryptedKey +} + +// ReadMessage parses an OpenPGP message that may be signed and/or encrypted. +// The given KeyRing should contain both public keys (for signature +// verification) and, possibly encrypted, private keys for decrypting. +// If config is nil, sensible defaults will be used. +func ReadMessage(r io.Reader, keyring KeyRing, prompt PromptFunction, config *packet.Config) (md *MessageDetails, err error) { + var p packet.Packet + + var symKeys []*packet.SymmetricKeyEncrypted + var pubKeys []keyEnvelopePair + var se *packet.SymmetricallyEncrypted + + packets := packet.NewReader(r) + md = new(MessageDetails) + md.IsEncrypted = true + + // The message, if encrypted, starts with a number of packets + // containing an encrypted decryption key. The decryption key is either + // encrypted to a public key, or with a passphrase. This loop + // collects these packets. +ParsePackets: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.SymmetricKeyEncrypted: + // This packet contains the decryption key encrypted with a passphrase. + md.IsSymmetricallyEncrypted = true + symKeys = append(symKeys, p) + case *packet.EncryptedKey: + // This packet contains the decryption key encrypted to a public key. + md.EncryptedToKeyIds = append(md.EncryptedToKeyIds, p.KeyId) + switch p.Algo { + case packet.PubKeyAlgoRSA, packet.PubKeyAlgoRSAEncryptOnly, packet.PubKeyAlgoElGamal: + break + default: + continue + } + var keys []Key + if p.KeyId == 0 { + keys = keyring.DecryptionKeys() + } else { + keys = keyring.KeysById(p.KeyId) + } + for _, k := range keys { + pubKeys = append(pubKeys, keyEnvelopePair{k, p}) + } + case *packet.SymmetricallyEncrypted: + se = p + break ParsePackets + case *packet.Compressed, *packet.LiteralData, *packet.OnePassSignature: + // This message isn't encrypted. + if len(symKeys) != 0 || len(pubKeys) != 0 { + return nil, errors.StructuralError("key material not followed by encrypted message") + } + packets.Unread(p) + return readSignedMessage(packets, nil, keyring) + } + } + + var candidates []Key + var decrypted io.ReadCloser + + // Now that we have the list of encrypted keys we need to decrypt at + // least one of them or, if we cannot, we need to call the prompt + // function so that it can decrypt a key or give us a passphrase. +FindKey: + for { + // See if any of the keys already have a private key available + candidates = candidates[:0] + candidateFingerprints := make(map[string]bool) + + for _, pk := range pubKeys { + if pk.key.PrivateKey == nil { + continue + } + if !pk.key.PrivateKey.Encrypted { + if len(pk.encryptedKey.Key) == 0 { + pk.encryptedKey.Decrypt(pk.key.PrivateKey, config) + } + if len(pk.encryptedKey.Key) == 0 { + continue + } + decrypted, err = se.Decrypt(pk.encryptedKey.CipherFunc, pk.encryptedKey.Key) + if err != nil && err != errors.ErrKeyIncorrect { + return nil, err + } + if decrypted != nil { + md.DecryptedWith = pk.key + break FindKey + } + } else { + fpr := string(pk.key.PublicKey.Fingerprint[:]) + if v := candidateFingerprints[fpr]; v { + continue + } + candidates = append(candidates, pk.key) + candidateFingerprints[fpr] = true + } + } + + if len(candidates) == 0 && len(symKeys) == 0 { + return nil, errors.ErrKeyIncorrect + } + + if prompt == nil { + return nil, errors.ErrKeyIncorrect + } + + passphrase, err := prompt(candidates, len(symKeys) != 0) + if err != nil { + return nil, err + } + + // Try the symmetric passphrase first + if len(symKeys) != 0 && passphrase != nil { + for _, s := range symKeys { + err = s.Decrypt(passphrase) + if err == nil && !s.Encrypted { + decrypted, err = se.Decrypt(s.CipherFunc, s.Key) + if err != nil && err != errors.ErrKeyIncorrect { + return nil, err + } + if decrypted != nil { + break FindKey + } + } + + } + } + } + + md.decrypted = decrypted + packets.Push(decrypted) + return readSignedMessage(packets, md, keyring) +} + +// readSignedMessage reads a possibly signed message if mdin is non-zero then +// that structure is updated and returned. Otherwise a fresh MessageDetails is +// used. +func readSignedMessage(packets *packet.Reader, mdin *MessageDetails, keyring KeyRing) (md *MessageDetails, err error) { + if mdin == nil { + mdin = new(MessageDetails) + } + md = mdin + + var p packet.Packet + var h hash.Hash + var wrappedHash hash.Hash +FindLiteralData: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.Compressed: + packets.Push(p.Body) + case *packet.OnePassSignature: + if !p.IsLast { + return nil, errors.UnsupportedError("nested signatures") + } + + h, wrappedHash, err = hashForSignature(p.Hash, p.SigType) + if err != nil { + md = nil + return + } + + md.IsSigned = true + md.SignedByKeyId = p.KeyId + keys := keyring.KeysByIdUsage(p.KeyId, packet.KeyFlagSign) + if len(keys) > 0 { + md.SignedBy = &keys[0] + } + case *packet.LiteralData: + md.LiteralData = p + break FindLiteralData + } + } + + if md.SignedBy != nil { + md.UnverifiedBody = &signatureCheckReader{packets, h, wrappedHash, md} + } else if md.decrypted != nil { + md.UnverifiedBody = checkReader{md} + } else { + md.UnverifiedBody = md.LiteralData.Body + } + + return md, nil +} + +// hashForSignature returns a pair of hashes that can be used to verify a +// signature. The signature may specify that the contents of the signed message +// should be preprocessed (i.e. to normalize line endings). Thus this function +// returns two hashes. The second should be used to hash the message itself and +// performs any needed preprocessing. +func hashForSignature(hashId crypto.Hash, sigType packet.SignatureType) (hash.Hash, hash.Hash, error) { + if !hashId.Available() { + return nil, nil, errors.UnsupportedError("hash not available: " + strconv.Itoa(int(hashId))) + } + h := hashId.New() + + switch sigType { + case packet.SigTypeBinary: + return h, h, nil + case packet.SigTypeText: + return h, NewCanonicalTextHash(h), nil + } + + return nil, nil, errors.UnsupportedError("unsupported signature type: " + strconv.Itoa(int(sigType))) +} + +// checkReader wraps an io.Reader from a LiteralData packet. When it sees EOF +// it closes the ReadCloser from any SymmetricallyEncrypted packet to trigger +// MDC checks. +type checkReader struct { + md *MessageDetails +} + +func (cr checkReader) Read(buf []byte) (n int, err error) { + n, err = cr.md.LiteralData.Body.Read(buf) + if err == io.EOF { + mdcErr := cr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + return +} + +// signatureCheckReader wraps an io.Reader from a LiteralData packet and hashes +// the data as it is read. When it sees an EOF from the underlying io.Reader +// it parses and checks a trailing Signature packet and triggers any MDC checks. +type signatureCheckReader struct { + packets *packet.Reader + h, wrappedHash hash.Hash + md *MessageDetails +} + +func (scr *signatureCheckReader) Read(buf []byte) (n int, err error) { + n, err = scr.md.LiteralData.Body.Read(buf) + scr.wrappedHash.Write(buf[:n]) + if err == io.EOF { + var p packet.Packet + p, scr.md.SignatureError = scr.packets.Next() + if scr.md.SignatureError != nil { + return + } + + var ok bool + if scr.md.Signature, ok = p.(*packet.Signature); !ok { + scr.md.SignatureError = errors.StructuralError("LiteralData not followed by Signature") + return + } + + scr.md.SignatureError = scr.md.SignedBy.PublicKey.VerifySignature(scr.h, scr.md.Signature) + + // The SymmetricallyEncrypted packet, if any, might have an + // unsigned hash of its own. In order to check this we need to + // close that Reader. + if scr.md.decrypted != nil { + mdcErr := scr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + } + return +} + +// CheckDetachedSignature takes a signed file and a detached signature and +// returns the signer if the signature is valid. If the signer isn't known, +// ErrUnknownIssuer is returned. +func CheckDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err error) { + p, err := packet.Read(signature) + if err != nil { + return + } + + var issuerKeyId uint64 + var hashFunc crypto.Hash + var sigType packet.SignatureType + + switch sig := p.(type) { + case *packet.Signature: + if sig.IssuerKeyId == nil { + return nil, errors.StructuralError("signature doesn't have an issuer") + } + issuerKeyId = *sig.IssuerKeyId + hashFunc = sig.Hash + sigType = sig.SigType + case *packet.SignatureV3: + issuerKeyId = sig.IssuerKeyId + hashFunc = sig.Hash + sigType = sig.SigType + default: + return nil, errors.StructuralError("non signature packet found") + } + + h, wrappedHash, err := hashForSignature(hashFunc, sigType) + if err != nil { + return + } + + _, err = io.Copy(wrappedHash, signed) + if err != nil && err != io.EOF { + return + } + + keys := keyring.KeysByIdUsage(issuerKeyId, packet.KeyFlagSign) + if len(keys) == 0 { + return nil, errors.ErrUnknownIssuer + } + + for _, key := range keys { + switch sig := p.(type) { + case *packet.Signature: + err = key.PublicKey.VerifySignature(h, sig) + case *packet.SignatureV3: + err = key.PublicKey.VerifySignatureV3(h, sig) + } + if err == nil { + return key.Entity, nil + } + } + + if err == nil { + err = errors.ErrUnknownIssuer + } + return nil, err +} + +// CheckArmoredDetachedSignature performs the same actions as +// CheckDetachedSignature but expects the signature to be armored. +func CheckArmoredDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err error) { + body, err := readArmored(signature, SignatureType) + if err != nil { + return + } + + return CheckDetachedSignature(keyring, signed, body) +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/read_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/read_test.go new file mode 100644 index 00000000000..c2720ae5e4e --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/read_test.go @@ -0,0 +1,416 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + _ "crypto/sha512" + "encoding/hex" + "golang.org/x/crypto/openpgp/errors" + "io" + "io/ioutil" + "strings" + "testing" +) + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func TestReadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestRereadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Errorf("error in initial parse: %s", err) + return + } + out := new(bytes.Buffer) + err = kring[0].Serialize(out) + if err != nil { + t.Errorf("error in serialization: %s", err) + return + } + kring, err = ReadKeyRing(out) + if err != nil { + t.Errorf("error in second parse: %s", err) + return + } + + if len(kring) != 1 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadPrivateKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B || kring[0].PrimaryKey == nil { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadDSAKey(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(dsaTestKeyHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 1 || uint32(kring[0].PrimaryKey.KeyId) != 0x0CCC0360 { + t.Errorf("bad parse: %#v", kring) + } +} + +func TestDSAHashTruncatation(t *testing.T) { + // dsaKeyWithSHA512 was generated with GnuPG and --cert-digest-algo + // SHA512 in order to require DSA hash truncation to verify correctly. + _, err := ReadKeyRing(readerFromHex(dsaKeyWithSHA512)) + if err != nil { + t.Error(err) + } +} + +func TestGetKeyById(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + keys := kring.KeysById(0xa34d7e18c20c31bb) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } + + keys = kring.KeysById(0xfd94408d4543314f) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } +} + +func checkSignedMessage(t *testing.T, signedHex, expected string) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + md, err := ReadMessage(readerFromHex(signedHex), kring, nil, nil) + if err != nil { + t.Error(err) + return + } + + if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) != 0 || md.IsSymmetricallyEncrypted { + t.Errorf("bad MessageDetails: %#v", md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("failed to validate: %s", md.SignatureError) + } +} + +func TestSignedMessage(t *testing.T) { + checkSignedMessage(t, signedMessageHex, signedInput) +} + +func TestTextSignedMessage(t *testing.T) { + checkSignedMessage(t, signedTextMessageHex, signedTextInput) +} + +var signedEncryptedMessageTests = []struct { + keyRingHex string + messageHex string + signedByKeyId uint64 + encryptedToKeyId uint64 +}{ + { + testKeys1And2PrivateHex, + signedEncryptedMessageHex, + 0xa34d7e18c20c31bb, + 0x2a67d68660df41c7, + }, + { + dsaElGamalTestKeysHex, + signedEncryptedMessage2Hex, + 0x33af447ccd759b09, + 0xcf6a7abcd43e3673, + }, +} + +func TestSignedEncryptedMessage(t *testing.T) { + for i, test := range signedEncryptedMessageTests { + expected := "Signed and encrypted message\n" + kring, _ := ReadKeyRing(readerFromHex(test.keyRingHex)) + prompt := func(keys []Key, symmetric bool) ([]byte, error) { + if symmetric { + t.Errorf("prompt: message was marked as symmetrically encrypted") + return nil, errors.ErrKeyIncorrect + } + + if len(keys) == 0 { + t.Error("prompt: no keys requested") + return nil, errors.ErrKeyIncorrect + } + + err := keys[0].PrivateKey.Decrypt([]byte("passphrase")) + if err != nil { + t.Errorf("prompt: error decrypting key: %s", err) + return nil, errors.ErrKeyIncorrect + } + + return nil, nil + } + + md, err := ReadMessage(readerFromHex(test.messageHex), kring, prompt, nil) + if err != nil { + t.Errorf("#%d: error reading message: %s", i, err) + return + } + + if !md.IsSigned || md.SignedByKeyId != test.signedByKeyId || md.SignedBy == nil || !md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) == 0 || md.EncryptedToKeyIds[0] != test.encryptedToKeyId { + t.Errorf("#%d: bad MessageDetails: %#v", i, md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("#%d: error reading UnverifiedBody: %s", i, err) + } + if string(contents) != expected { + t.Errorf("#%d: bad UnverifiedBody got:%s want:%s", i, string(contents), expected) + } + + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("#%d: failed to validate: %s", i, md.SignatureError) + } + } +} + +func TestUnspecifiedRecipient(t *testing.T) { + expected := "Recipient unspecified\n" + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + + md, err := ReadMessage(readerFromHex(recipientUnspecifiedHex), kring, nil, nil) + if err != nil { + t.Errorf("error reading message: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } +} + +func TestSymmetricallyEncrypted(t *testing.T) { + expected := "Symmetrically encrypted.\n" + + prompt := func(keys []Key, symmetric bool) ([]byte, error) { + if len(keys) != 0 { + t.Errorf("prompt: len(keys) = %d (want 0)", len(keys)) + } + + if !symmetric { + t.Errorf("symmetric is not set") + } + + return []byte("password"), nil + } + + md, err := ReadMessage(readerFromHex(symmetricallyEncryptedCompressedHex), nil, prompt, nil) + if err != nil { + t.Errorf("ReadMessage: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("ReadAll: %s", err) + } + + expectedCreationTime := uint32(1295992998) + if md.LiteralData.Time != expectedCreationTime { + t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreationTime) + } + + if string(contents) != expected { + t.Errorf("contents got: %s want: %s", string(contents), expected) + } +} + +func testDetachedSignature(t *testing.T, kring KeyRing, signature io.Reader, sigInput, tag string, expectedSignerKeyId uint64) { + signed := bytes.NewBufferString(sigInput) + signer, err := CheckDetachedSignature(kring, signed, signature) + if err != nil { + t.Errorf("%s: signature error: %s", tag, err) + return + } + if signer == nil { + t.Errorf("%s: signer is nil", tag) + return + } + if signer.PrimaryKey.KeyId != expectedSignerKeyId { + t.Errorf("%s: wrong signer got:%x want:%x", tag, signer.PrimaryKey.KeyId, expectedSignerKeyId) + } +} + +func TestDetachedSignature(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureHex), signedInput, "binary", testKey1KeyId) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureTextHex), signedInput, "text", testKey1KeyId) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureV3TextHex), signedInput, "v3", testKey1KeyId) + + incorrectSignedInput := signedInput + "X" + _, err := CheckDetachedSignature(kring, bytes.NewBufferString(incorrectSignedInput), readerFromHex(detachedSignatureHex)) + if err == nil { + t.Fatal("CheckDetachedSignature returned without error for bad signature") + } + if err == errors.ErrUnknownIssuer { + t.Fatal("CheckDetachedSignature returned ErrUnknownIssuer when the signer was known, but the signature invalid") + } +} + +func TestDetachedSignatureDSA(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(dsaTestKeyHex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureDSAHex), signedInput, "binary", testKey3KeyId) +} + +func testHashFunctionError(t *testing.T, signatureHex string) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + _, err := CheckDetachedSignature(kring, nil, readerFromHex(signatureHex)) + if err == nil { + t.Fatal("Packet with bad hash type was correctly parsed") + } + unsupported, ok := err.(errors.UnsupportedError) + if !ok { + t.Fatalf("Unexpected class of error: %s", err) + } + if !strings.Contains(string(unsupported), "hash ") { + t.Fatalf("Unexpected error: %s", err) + } +} + +func TestUnknownHashFunction(t *testing.T) { + // unknownHashFunctionHex contains a signature packet with hash + // function type 153 (which isn't a real hash function id). + testHashFunctionError(t, unknownHashFunctionHex) +} + +func TestMissingHashFunction(t *testing.T) { + // missingHashFunctionHex contains a signature packet that uses + // RIPEMD160, which isn't compiled in. + testHashFunctionError(t, missingHashFunctionHex) +} + +func TestReadingArmoredPrivateKey(t *testing.T) { + el, err := ReadArmoredKeyRing(bytes.NewBufferString(armoredPrivateKeyBlock)) + if err != nil { + t.Error(err) + } + if len(el) != 1 { + t.Errorf("got %d entities, wanted 1\n", len(el)) + } +} + +func TestNoArmoredData(t *testing.T) { + _, err := ReadArmoredKeyRing(bytes.NewBufferString("foo")) + if _, ok := err.(errors.InvalidArgumentError); !ok { + t.Errorf("error was not an InvalidArgumentError: %s", err) + } +} + +const testKey1KeyId = 0xA34D7E18C20C31BB +const testKey3KeyId = 0x338934250CCC0360 + +const signedInput = "Signed message\nline 2\nline 3\n" +const signedTextInput = "Signed message\r\nline 2\r\nline 3\r\n" + +const recipientUnspecifiedHex = "848c0300000000000000000103ff62d4d578d03cf40c3da998dfe216c074fa6ddec5e31c197c9666ba292830d91d18716a80f699f9d897389a90e6d62d0238f5f07a5248073c0f24920e4bc4a30c2d17ee4e0cae7c3d4aaa4e8dced50e3010a80ee692175fa0385f62ecca4b56ee6e9980aa3ec51b61b077096ac9e800edaf161268593eedb6cc7027ff5cb32745d250010d407a6221ae22ef18469b444f2822478c4d190b24d36371a95cb40087cdd42d9399c3d06a53c0673349bfb607927f20d1e122bde1e2bf3aa6cae6edf489629bcaa0689539ae3b718914d88ededc3b" + +const detachedSignatureHex = "889c04000102000605024d449cd1000a0910a34d7e18c20c31bb167603ff57718d09f28a519fdc7b5a68b6a3336da04df85e38c5cd5d5bd2092fa4629848a33d85b1729402a2aab39c3ac19f9d573f773cc62c264dc924c067a79dfd8a863ae06c7c8686120760749f5fd9b1e03a64d20a7df3446ddc8f0aeadeaeba7cbaee5c1e366d65b6a0c6cc749bcb912d2f15013f812795c2e29eb7f7b77f39ce77" + +const detachedSignatureTextHex = "889c04010102000605024d449d21000a0910a34d7e18c20c31bbc8c60400a24fbef7342603a41cb1165767bd18985d015fb72fe05db42db36cfb2f1d455967f1e491194fbf6cf88146222b23bf6ffbd50d17598d976a0417d3192ff9cc0034fd00f287b02e90418bbefe609484b09231e4e7a5f3562e199bf39909ab5276c4d37382fe088f6b5c3426fc1052865da8b3ab158672d58b6264b10823dc4b39" + +const detachedSignatureV3TextHex = "8900950305005255c25ca34d7e18c20c31bb0102bb3f04009f6589ef8a028d6e54f6eaf25432e590d31c3a41f4710897585e10c31e5e332c7f9f409af8512adceaff24d0da1474ab07aa7bce4f674610b010fccc5b579ae5eb00a127f272fb799f988ab8e4574c141da6dbfecfef7e6b2c478d9a3d2551ba741f260ee22bec762812f0053e05380bfdd55ad0f22d8cdf71b233fe51ae8a24" + +const detachedSignatureDSAHex = "884604001102000605024d6c4eac000a0910338934250ccc0360f18d00a087d743d6405ed7b87755476629600b8b694a39e900a0abff8126f46faf1547c1743c37b21b4ea15b8f83" + +const testKeys1And2Hex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b0020003b88d044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f0011010001889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab0020003988d044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b0020003b88d044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020003" + +const testKeys1And2PrivateHex = "9501d8044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd00110100010003ff4d91393b9a8e3430b14d6209df42f98dc927425b881f1209f319220841273a802a97c7bdb8b3a7740b3ab5866c4d1d308ad0d3a79bd1e883aacf1ac92dfe720285d10d08752a7efe3c609b1d00f17f2805b217be53999a7da7e493bfc3e9618fd17018991b8128aea70a05dbce30e4fbe626aa45775fa255dd9177aabf4df7cf0200c1ded12566e4bc2bb590455e5becfb2e2c9796482270a943343a7835de41080582c2be3caf5981aa838140e97afa40ad652a0b544f83eb1833b0957dce26e47b0200eacd6046741e9ce2ec5beb6fb5e6335457844fb09477f83b050a96be7da043e17f3a9523567ed40e7a521f818813a8b8a72209f1442844843ccc7eb9805442570200bdafe0438d97ac36e773c7162028d65844c4d463e2420aa2228c6e50dc2743c3d6c72d0d782a5173fe7be2169c8a9f4ef8a7cf3e37165e8c61b89c346cdc6c1799d2b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b00200009d01d8044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f00110100010003fd17a7490c22a79c59281fb7b20f5e6553ec0c1637ae382e8adaea295f50241037f8997cf42c1ce26417e015091451b15424b2c59eb8d4161b0975630408e394d3b00f88d4b4e18e2cc85e8251d4753a27c639c83f5ad4a571c4f19d7cd460b9b73c25ade730c99df09637bd173d8e3e981ac64432078263bb6dc30d3e974150dd0200d0ee05be3d4604d2146fb0457f31ba17c057560785aa804e8ca5530a7cd81d3440d0f4ba6851efcfd3954b7e68908fc0ba47f7ac37bf559c6c168b70d3a7c8cd0200da1c677c4bce06a068070f2b3733b0a714e88d62aa3f9a26c6f5216d48d5c2b5624144f3807c0df30be66b3268eeeca4df1fbded58faf49fc95dc3c35f134f8b01fd1396b6c0fc1b6c4f0eb8f5e44b8eace1e6073e20d0b8bc5385f86f1cf3f050f66af789f3ef1fc107b7f4421e19e0349c730c68f0a226981f4e889054fdb4dc149e8e889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab00200009501fe044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001fe030302e9030f3c783e14856063f16938530e148bc57a7aa3f3e4f90df9dceccdc779bc0835e1ad3d006e4a8d7b36d08b8e0de5a0d947254ecfbd22037e6572b426bcfdc517796b224b0036ff90bc574b5509bede85512f2eefb520fb4b02aa523ba739bff424a6fe81c5041f253f8d757e69a503d3563a104d0d49e9e890b9d0c26f96b55b743883b472caa7050c4acfd4a21f875bdf1258d88bd61224d303dc9df77f743137d51e6d5246b88c406780528fd9a3e15bab5452e5b93970d9dcc79f48b38651b9f15bfbcf6da452837e9cc70683d1bdca94507870f743e4ad902005812488dd342f836e72869afd00ce1850eea4cfa53ce10e3608e13d3c149394ee3cbd0e23d018fcbcb6e2ec5a1a22972d1d462ca05355d0d290dd2751e550d5efb38c6c89686344df64852bf4ff86638708f644e8ec6bd4af9b50d8541cb91891a431326ab2e332faa7ae86cfb6e0540aa63160c1e5cdd5a4add518b303fff0a20117c6bc77f7cfbaf36b04c865c6c2b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b00200009d01fe044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001fe030302e9030f3c783e148560f936097339ae381d63116efcf802ff8b1c9360767db5219cc987375702a4123fd8657d3e22700f23f95020d1b261eda5257e9a72f9a918e8ef22dd5b3323ae03bbc1923dd224db988cadc16acc04b120a9f8b7e84da9716c53e0334d7b66586ddb9014df604b41be1e960dcfcbc96f4ed150a1a0dd070b9eb14276b9b6be413a769a75b519a53d3ecc0c220e85cd91ca354d57e7344517e64b43b6e29823cbd87eae26e2b2e78e6dedfbb76e3e9f77bcb844f9a8932eb3db2c3f9e44316e6f5d60e9e2a56e46b72abe6b06dc9a31cc63f10023d1f5e12d2a3ee93b675c96f504af0001220991c88db759e231b3320dcedf814dcf723fd9857e3d72d66a0f2af26950b915abdf56c1596f46a325bf17ad4810d3535fb02a259b247ac3dbd4cc3ecf9c51b6c07cebb009c1506fba0a89321ec8683e3fd009a6e551d50243e2d5092fefb3321083a4bad91320dc624bd6b5dddf93553e3d53924c05bfebec1fb4bd47e89a1a889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020000" + +const dsaElGamalTestKeysHex = "9501e1044dfcb16a110400aa3e5c1a1f43dd28c2ffae8abf5cfce555ee874134d8ba0a0f7b868ce2214beddc74e5e1e21ded354a95d18acdaf69e5e342371a71fbb9093162e0c5f3427de413a7f2c157d83f5cd2f9d791256dc4f6f0e13f13c3302af27f2384075ab3021dff7a050e14854bbde0a1094174855fc02f0bae8e00a340d94a1f22b32e48485700a0cec672ac21258fb95f61de2ce1af74b2c4fa3e6703ff698edc9be22c02ae4d916e4fa223f819d46582c0516235848a77b577ea49018dcd5e9e15cff9dbb4663a1ae6dd7580fa40946d40c05f72814b0f88481207e6c0832c3bded4853ebba0a7e3bd8e8c66df33d5a537cd4acf946d1080e7a3dcea679cb2b11a72a33a2b6a9dc85f466ad2ddf4c3db6283fa645343286971e3dd700703fc0c4e290d45767f370831a90187e74e9972aae5bff488eeff7d620af0362bfb95c1a6c3413ab5d15a2e4139e5d07a54d72583914661ed6a87cce810be28a0aa8879a2dd39e52fb6fe800f4f181ac7e328f740cde3d09a05cecf9483e4cca4253e60d4429ffd679d9996a520012aad119878c941e3cf151459873bdfc2a9563472fe0303027a728f9feb3b864260a1babe83925ce794710cfd642ee4ae0e5b9d74cee49e9c67b6cd0ea5dfbb582132195a121356a1513e1bca73e5b80c58c7ccb4164453412f456c47616d616c2054657374204b65792031886204131102002205024dfcb16a021b03060b090807030206150802090a0b0416020301021e01021780000a091033af447ccd759b09fadd00a0b8fd6f5a790bad7e9f2dbb7632046dc4493588db009c087c6a9ba9f7f49fab221587a74788c00db4889ab00200009d0157044dfcb16a1004008dec3f9291205255ccff8c532318133a6840739dd68b03ba942676f9038612071447bf07d00d559c5c0875724ea16a4c774f80d8338b55fca691a0522e530e604215b467bbc9ccfd483a1da99d7bc2648b4318fdbd27766fc8bfad3fddb37c62b8ae7ccfe9577e9b8d1e77c1d417ed2c2ef02d52f4da11600d85d3229607943700030503ff506c94c87c8cab778e963b76cf63770f0a79bf48fb49d3b4e52234620fc9f7657f9f8d56c96a2b7c7826ae6b57ebb2221a3fe154b03b6637cea7e6d98e3e45d87cf8dc432f723d3d71f89c5192ac8d7290684d2c25ce55846a80c9a7823f6acd9bb29fa6cd71f20bc90eccfca20451d0c976e460e672b000df49466408d527affe0303027a728f9feb3b864260abd761730327bca2aaa4ea0525c175e92bf240682a0e83b226f97ecb2e935b62c9a133858ce31b271fa8eb41f6a1b3cd72a63025ce1a75ee4180dcc284884904181102000905024dfcb16a021b0c000a091033af447ccd759b09dd0b009e3c3e7296092c81bee5a19929462caaf2fff3ae26009e218c437a2340e7ea628149af1ec98ec091a43992b00200009501e1044dfcb1be1104009f61faa61aa43df75d128cbe53de528c4aec49ce9360c992e70c77072ad5623de0a3a6212771b66b39a30dad6781799e92608316900518ec01184a85d872365b7d2ba4bacfb5882ea3c2473d3750dc6178cc1cf82147fb58caa28b28e9f12f6d1efcb0534abed644156c91cca4ab78834268495160b2400bc422beb37d237c2300a0cac94911b6d493bda1e1fbc6feeca7cb7421d34b03fe22cec6ccb39675bb7b94a335c2b7be888fd3906a1125f33301d8aa6ec6ee6878f46f73961c8d57a3e9544d8ef2a2cbfd4d52da665b1266928cfe4cb347a58c412815f3b2d2369dec04b41ac9a71cc9547426d5ab941cccf3b18575637ccfb42df1a802df3cfe0a999f9e7109331170e3a221991bf868543960f8c816c28097e503fe319db10fb98049f3a57d7c80c420da66d56f3644371631fad3f0ff4040a19a4fedc2d07727a1b27576f75a4d28c47d8246f27071e12d7a8de62aad216ddbae6aa02efd6b8a3e2818cda48526549791ab277e447b3a36c57cefe9b592f5eab73959743fcc8e83cbefec03a329b55018b53eec196765ae40ef9e20521a603c551efe0303020950d53a146bf9c66034d00c23130cce95576a2ff78016ca471276e8227fb30b1ffbd92e61804fb0c3eff9e30b1a826ee8f3e4730b4d86273ca977b4164453412f456c47616d616c2054657374204b65792032886204131102002205024dfcb1be021b03060b090807030206150802090a0b0416020301021e01021780000a0910a86bf526325b21b22bd9009e34511620415c974750a20df5cb56b182f3b48e6600a0a9466cb1a1305a84953445f77d461593f1d42bc1b00200009d0157044dfcb1be1004009565a951da1ee87119d600c077198f1c1bceb0f7aa54552489298e41ff788fa8f0d43a69871f0f6f77ebdfb14a4260cf9fbeb65d5844b4272a1904dd95136d06c3da745dc46327dd44a0f16f60135914368c8039a34033862261806bb2c5ce1152e2840254697872c85441ccb7321431d75a747a4bfb1d2c66362b51ce76311700030503fc0ea76601c196768070b7365a200e6ddb09307f262d5f39eec467b5f5784e22abdf1aa49226f59ab37cb49969d8f5230ea65caf56015abda62604544ed526c5c522bf92bed178a078789f6c807b6d34885688024a5bed9e9f8c58d11d4b82487b44c5f470c5606806a0443b79cadb45e0f897a561a53f724e5349b9267c75ca17fe0303020950d53a146bf9c660bc5f4ce8f072465e2d2466434320c1e712272fafc20e342fe7608101580fa1a1a367e60486a7cd1246b7ef5586cf5e10b32762b710a30144f12dd17dd4884904181102000905024dfcb1be021b0c000a0910a86bf526325b21b2904c00a0b2b66b4b39ccffda1d10f3ea8d58f827e30a8b8e009f4255b2d8112a184e40cde43a34e8655ca7809370b0020000" + +const signedMessageHex = "a3019bc0cbccc0c4b8d8b74ee2108fe16ec6d3ca490cbe362d3f8333d3f352531472538b8b13d353b97232f352158c20943157c71c16064626063656269052062e4e01987e9b6fccff4b7df3a34c534b23e679cbec3bc0f8f6e64dfb4b55fe3f8efa9ce110ddb5cd79faf1d753c51aecfa669f7e7aa043436596cccc3359cb7dd6bbe9ecaa69e5989d9e57209571edc0b2fa7f57b9b79a64ee6e99ce1371395fee92fec2796f7b15a77c386ff668ee27f6d38f0baa6c438b561657377bf6acff3c5947befd7bf4c196252f1d6e5c524d0300" + +const signedTextMessageHex = "a3019bc0cbccc8c4b8d8b74ee2108fe16ec6d36a250cbece0c178233d3f352531472538b8b13d35379b97232f352158ca0b4312f57c71c1646462606365626906a062e4e019811591798ff99bf8afee860b0d8a8c2a85c3387e3bcf0bb3b17987f2bbcfab2aa526d930cbfd3d98757184df3995c9f3e7790e36e3e9779f06089d4c64e9e47dd6202cb6e9bc73c5d11bb59fbaf89d22d8dc7cf199ddf17af96e77c5f65f9bbed56f427bd8db7af37f6c9984bf9385efaf5f184f986fb3e6adb0ecfe35bbf92d16a7aa2a344fb0bc52fb7624f0200" + +const signedEncryptedMessageHex = "848c032a67d68660df41c70103ff5789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8d2c03b018bd210b1d3791e1aba74b0f1034e122ab72e760492c192383cf5e20b5628bd043272d63df9b923f147eb6091cd897553204832aba48fec54aa447547bb16305a1024713b90e77fd0065f1918271947549205af3c74891af22ee0b56cd29bfec6d6e351901cd4ab3ece7c486f1e32a792d4e474aed98ee84b3f591c7dff37b64e0ecd68fd036d517e412dcadf85840ce184ad7921ad446c4ee28db80447aea1ca8d4f574db4d4e37688158ddd19e14ee2eab4873d46947d65d14a23e788d912cf9a19624ca7352469b72a83866b7c23cb5ace3deab3c7018061b0ba0f39ed2befe27163e5083cf9b8271e3e3d52cc7ad6e2a3bd81d4c3d7022f8d" + +const signedEncryptedMessage2Hex = "85010e03cf6a7abcd43e36731003fb057f5495b79db367e277cdbe4ab90d924ddee0c0381494112ff8c1238fb0184af35d1731573b01bc4c55ecacd2aafbe2003d36310487d1ecc9ac994f3fada7f9f7f5c3a64248ab7782906c82c6ff1303b69a84d9a9529c31ecafbcdb9ba87e05439897d87e8a2a3dec55e14df19bba7f7bd316291c002ae2efd24f83f9e3441203fc081c0c23dc3092a454ca8a082b27f631abf73aca341686982e8fbda7e0e7d863941d68f3de4a755c2964407f4b5e0477b3196b8c93d551dd23c8beef7d0f03fbb1b6066f78907faf4bf1677d8fcec72651124080e0b7feae6b476e72ab207d38d90b958759fdedfc3c6c35717c9dbfc979b3cfbbff0a76d24a5e57056bb88acbd2a901ef64bc6e4db02adc05b6250ff378de81dca18c1910ab257dff1b9771b85bb9bbe0a69f5989e6d1710a35e6dfcceb7d8fb5ccea8db3932b3d9ff3fe0d327597c68b3622aec8e3716c83a6c93f497543b459b58ba504ed6bcaa747d37d2ca746fe49ae0a6ce4a8b694234e941b5159ff8bd34b9023da2814076163b86f40eed7c9472f81b551452d5ab87004a373c0172ec87ea6ce42ccfa7dbdad66b745496c4873d8019e8c28d6b3" + +const symmetricallyEncryptedCompressedHex = "8c0d04030302eb4a03808145d0d260c92f714339e13de5a79881216431925bf67ee2898ea61815f07894cd0703c50d0a76ef64d482196f47a8bc729af9b80bb6" + +const dsaTestKeyHex = "9901a2044d6c49de110400cb5ce438cf9250907ac2ba5bf6547931270b89f7c4b53d9d09f4d0213a5ef2ec1f26806d3d259960f872a4a102ef1581ea3f6d6882d15134f21ef6a84de933cc34c47cc9106efe3bd84c6aec12e78523661e29bc1a61f0aab17fa58a627fd5fd33f5149153fbe8cd70edf3d963bc287ef875270ff14b5bfdd1bca4483793923b00a0fe46d76cb6e4cbdc568435cd5480af3266d610d303fe33ae8273f30a96d4d34f42fa28ce1112d425b2e3bf7ea553d526e2db6b9255e9dc7419045ce817214d1a0056dbc8d5289956a4b1b69f20f1105124096e6a438f41f2e2495923b0f34b70642607d45559595c7fe94d7fa85fc41bf7d68c1fd509ebeaa5f315f6059a446b9369c277597e4f474a9591535354c7e7f4fd98a08aa60400b130c24ff20bdfbf683313f5daebf1c9b34b3bdadfc77f2ddd72ee1fb17e56c473664bc21d66467655dd74b9005e3a2bacce446f1920cd7017231ae447b67036c9b431b8179deacd5120262d894c26bc015bffe3d827ba7087ad9b700d2ca1f6d16cc1786581e5dd065f293c31209300f9b0afcc3f7c08dd26d0a22d87580b4db41054657374204b65792033202844534129886204131102002205024d6c49de021b03060b090807030206150802090a0b0416020301021e01021780000a0910338934250ccc03607e0400a0bdb9193e8a6b96fc2dfc108ae848914b504481f100a09c4dc148cb693293a67af24dd40d2b13a9e36794" + +const dsaTestKeyPrivateHex = "9501bb044d6c49de110400cb5ce438cf9250907ac2ba5bf6547931270b89f7c4b53d9d09f4d0213a5ef2ec1f26806d3d259960f872a4a102ef1581ea3f6d6882d15134f21ef6a84de933cc34c47cc9106efe3bd84c6aec12e78523661e29bc1a61f0aab17fa58a627fd5fd33f5149153fbe8cd70edf3d963bc287ef875270ff14b5bfdd1bca4483793923b00a0fe46d76cb6e4cbdc568435cd5480af3266d610d303fe33ae8273f30a96d4d34f42fa28ce1112d425b2e3bf7ea553d526e2db6b9255e9dc7419045ce817214d1a0056dbc8d5289956a4b1b69f20f1105124096e6a438f41f2e2495923b0f34b70642607d45559595c7fe94d7fa85fc41bf7d68c1fd509ebeaa5f315f6059a446b9369c277597e4f474a9591535354c7e7f4fd98a08aa60400b130c24ff20bdfbf683313f5daebf1c9b34b3bdadfc77f2ddd72ee1fb17e56c473664bc21d66467655dd74b9005e3a2bacce446f1920cd7017231ae447b67036c9b431b8179deacd5120262d894c26bc015bffe3d827ba7087ad9b700d2ca1f6d16cc1786581e5dd065f293c31209300f9b0afcc3f7c08dd26d0a22d87580b4d00009f592e0619d823953577d4503061706843317e4fee083db41054657374204b65792033202844534129886204131102002205024d6c49de021b03060b090807030206150802090a0b0416020301021e01021780000a0910338934250ccc03607e0400a0bdb9193e8a6b96fc2dfc108ae848914b504481f100a09c4dc148cb693293a67af24dd40d2b13a9e36794" + +const armoredPrivateKeyBlock = `-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBE2rFNoBBADFwqWQIW/DSqcB4yCQqnAFTJ27qS5AnB46ccAdw3u4Greeu3Bp +idpoHdjULy7zSKlwR1EA873dO/k/e11Ml3dlAFUinWeejWaK2ugFP6JjiieSsrKn +vWNicdCS4HTWn0X4sjl0ZiAygw6GNhqEQ3cpLeL0g8E9hnYzJKQ0LWJa0QARAQAB +AAP/TB81EIo2VYNmTq0pK1ZXwUpxCrvAAIG3hwKjEzHcbQznsjNvPUihZ+NZQ6+X +0HCfPAdPkGDCLCb6NavcSW+iNnLTrdDnSI6+3BbIONqWWdRDYJhqZCkqmG6zqSfL +IdkJgCw94taUg5BWP/AAeQrhzjChvpMQTVKQL5mnuZbUCeMCAN5qrYMP2S9iKdnk +VANIFj7656ARKt/nf4CBzxcpHTyB8+d2CtPDKCmlJP6vL8t58Jmih+kHJMvC0dzn +gr5f5+sCAOOe5gt9e0am7AvQWhdbHVfJU0TQJx+m2OiCJAqGTB1nvtBLHdJnfdC9 +TnXXQ6ZXibqLyBies/xeY2sCKL5qtTMCAKnX9+9d/5yQxRyrQUHt1NYhaXZnJbHx +q4ytu0eWz+5i68IYUSK69jJ1NWPM0T6SkqpB3KCAIv68VFm9PxqG1KmhSrQIVGVz +dCBLZXmIuAQTAQIAIgUCTasU2gIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AA +CgkQO9o98PRieSoLhgQAkLEZex02Qt7vGhZzMwuN0R22w3VwyYyjBx+fM3JFETy1 +ut4xcLJoJfIaF5ZS38UplgakHG0FQ+b49i8dMij0aZmDqGxrew1m4kBfjXw9B/v+ +eIqpODryb6cOSwyQFH0lQkXC040pjq9YqDsO5w0WYNXYKDnzRV0p4H1pweo2VDid +AdgETasU2gEEAN46UPeWRqKHvA99arOxee38fBt2CI08iiWyI8T3J6ivtFGixSqV +bRcPxYO/qLpVe5l84Nb3X71GfVXlc9hyv7CD6tcowL59hg1E/DC5ydI8K8iEpUmK +/UnHdIY5h8/kqgGxkY/T/hgp5fRQgW1ZoZxLajVlMRZ8W4tFtT0DeA+JABEBAAEA +A/0bE1jaaZKj6ndqcw86jd+QtD1SF+Cf21CWRNeLKnUds4FRRvclzTyUMuWPkUeX +TaNNsUOFqBsf6QQ2oHUBBK4VCHffHCW4ZEX2cd6umz7mpHW6XzN4DECEzOVksXtc +lUC1j4UB91DC/RNQqwX1IV2QLSwssVotPMPqhOi0ZLNY7wIA3n7DWKInxYZZ4K+6 +rQ+POsz6brEoRHwr8x6XlHenq1Oki855pSa1yXIARoTrSJkBtn5oI+f8AzrnN0BN +oyeQAwIA/7E++3HDi5aweWrViiul9cd3rcsS0dEnksPhvS0ozCJiHsq/6GFmy7J8 +QSHZPteedBnZyNp5jR+H7cIfVN3KgwH/Skq4PsuPhDq5TKK6i8Pc1WW8MA6DXTdU +nLkX7RGmMwjC0DBf7KWAlPjFaONAX3a8ndnz//fy1q7u2l9AZwrj1qa1iJ8EGAEC +AAkFAk2rFNoCGwwACgkQO9o98PRieSo2/QP/WTzr4ioINVsvN1akKuekmEMI3LAp +BfHwatufxxP1U+3Si/6YIk7kuPB9Hs+pRqCXzbvPRrI8NHZBmc8qIGthishdCYad +AHcVnXjtxrULkQFGbGvhKURLvS9WnzD/m1K2zzwxzkPTzT9/Yf06O6Mal5AdugPL +VrM0m72/jnpKo04= +=zNCn +-----END PGP PRIVATE KEY BLOCK-----` + +const dsaKeyWithSHA512 = `9901a2044f04b07f110400db244efecc7316553ee08d179972aab87bb1214de7692593fcf5b6feb1c80fba268722dd464748539b85b81d574cd2d7ad0ca2444de4d849b8756bad7768c486c83a824f9bba4af773d11742bdfb4ac3b89ef8cc9452d4aad31a37e4b630d33927bff68e879284a1672659b8b298222fc68f370f3e24dccacc4a862442b9438b00a0ea444a24088dc23e26df7daf8f43cba3bffc4fe703fe3d6cd7fdca199d54ed8ae501c30e3ec7871ea9cdd4cf63cfe6fc82281d70a5b8bb493f922cd99fba5f088935596af087c8d818d5ec4d0b9afa7f070b3d7c1dd32a84fca08d8280b4890c8da1dde334de8e3cad8450eed2a4a4fcc2db7b8e5528b869a74a7f0189e11ef097ef1253582348de072bb07a9fa8ab838e993cef0ee203ff49298723e2d1f549b00559f886cd417a41692ce58d0ac1307dc71d85a8af21b0cf6eaa14baf2922d3a70389bedf17cc514ba0febbd107675a372fe84b90162a9e88b14d4b1c6be855b96b33fb198c46f058568817780435b6936167ebb3724b680f32bf27382ada2e37a879b3d9de2abe0c3f399350afd1ad438883f4791e2e3b4184453412068617368207472756e636174696f6e207465737488620413110a002205024f04b07f021b03060b090807030206150802090a0b0416020301021e01021780000a0910ef20e0cefca131581318009e2bf3bf047a44d75a9bacd00161ee04d435522397009a03a60d51bd8a568c6c021c8d7cf1be8d990d6417b0020003` + +const unknownHashFunctionHex = `8a00000040040001990006050253863c24000a09103b4fe6acc0b21f32ffff01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101` + +const missingHashFunctionHex = `8a00000040040001030006050253863c24000a09103b4fe6acc0b21f32ffff01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101` diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/s2k/s2k.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/s2k/s2k.go new file mode 100644 index 00000000000..54164214797 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/s2k/s2k.go @@ -0,0 +1,273 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package s2k implements the various OpenPGP string-to-key transforms as +// specified in RFC 4800 section 3.7.1. +package s2k + +import ( + "crypto" + "hash" + "io" + "strconv" + + "golang.org/x/crypto/openpgp/errors" +) + +// Config collects configuration parameters for s2k key-stretching +// transformatioms. A nil *Config is valid and results in all default +// values. Currently, Config is used only by the Serialize function in +// this package. +type Config struct { + // Hash is the default hash function to be used. If + // nil, SHA1 is used. + Hash crypto.Hash + // S2KCount is only used for symmetric encryption. It + // determines the strength of the passphrase stretching when + // the said passphrase is hashed to produce a key. S2KCount + // should be between 1024 and 65011712, inclusive. If Config + // is nil or S2KCount is 0, the value 65536 used. Not all + // values in the above range can be represented. S2KCount will + // be rounded up to the next representable value if it cannot + // be encoded exactly. When set, it is strongly encrouraged to + // use a value that is at least 65536. See RFC 4880 Section + // 3.7.1.3. + S2KCount int +} + +func (c *Config) hash() crypto.Hash { + if c == nil || uint(c.Hash) == 0 { + // SHA1 is the historical default in this package. + return crypto.SHA1 + } + + return c.Hash +} + +func (c *Config) encodedCount() uint8 { + if c == nil || c.S2KCount == 0 { + return 96 // The common case. Correspoding to 65536 + } + + i := c.S2KCount + switch { + // Behave like GPG. Should we make 65536 the lowest value used? + case i < 1024: + i = 1024 + case i > 65011712: + i = 65011712 + } + + return encodeCount(i) +} + +// encodeCount converts an iterative "count" in the range 1024 to +// 65011712, inclusive, to an encoded count. The return value is the +// octet that is actually stored in the GPG file. encodeCount panics +// if i is not in the above range (encodedCount above takes care to +// pass i in the correct range). See RFC 4880 Section 3.7.7.1. +func encodeCount(i int) uint8 { + if i < 1024 || i > 65011712 { + panic("count arg i outside the required range") + } + + for encoded := 0; encoded < 256; encoded++ { + count := decodeCount(uint8(encoded)) + if count >= i { + return uint8(encoded) + } + } + + return 255 +} + +// decodeCount returns the s2k mode 3 iterative "count" corresponding to +// the encoded octet c. +func decodeCount(c uint8) int { + return (16 + int(c&15)) << (uint32(c>>4) + 6) +} + +// Simple writes to out the result of computing the Simple S2K function (RFC +// 4880, section 3.7.1.1) using the given hash and input passphrase. +func Simple(out []byte, h hash.Hash, in []byte) { + Salted(out, h, in, nil) +} + +var zero [1]byte + +// Salted writes to out the result of computing the Salted S2K function (RFC +// 4880, section 3.7.1.2) using the given hash, input passphrase and salt. +func Salted(out []byte, h hash.Hash, in []byte, salt []byte) { + done := 0 + var digest []byte + + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + h.Write(salt) + h.Write(in) + digest = h.Sum(digest[:0]) + n := copy(out[done:], digest) + done += n + } +} + +// Iterated writes to out the result of computing the Iterated and Salted S2K +// function (RFC 4880, section 3.7.1.3) using the given hash, input passphrase, +// salt and iteration count. +func Iterated(out []byte, h hash.Hash, in []byte, salt []byte, count int) { + combined := make([]byte, len(in)+len(salt)) + copy(combined, salt) + copy(combined[len(salt):], in) + + if count < len(combined) { + count = len(combined) + } + + done := 0 + var digest []byte + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + written := 0 + for written < count { + if written+len(combined) > count { + todo := count - written + h.Write(combined[:todo]) + written = count + } else { + h.Write(combined) + written += len(combined) + } + } + digest = h.Sum(digest[:0]) + n := copy(out[done:], digest) + done += n + } +} + +// Parse reads a binary specification for a string-to-key transformation from r +// and returns a function which performs that transform. +func Parse(r io.Reader) (f func(out, in []byte), err error) { + var buf [9]byte + + _, err = io.ReadFull(r, buf[:2]) + if err != nil { + return + } + + hash, ok := HashIdToHash(buf[1]) + if !ok { + return nil, errors.UnsupportedError("hash for S2K function: " + strconv.Itoa(int(buf[1]))) + } + if !hash.Available() { + return nil, errors.UnsupportedError("hash not available: " + strconv.Itoa(int(hash))) + } + h := hash.New() + + switch buf[0] { + case 0: + f := func(out, in []byte) { + Simple(out, h, in) + } + return f, nil + case 1: + _, err = io.ReadFull(r, buf[:8]) + if err != nil { + return + } + f := func(out, in []byte) { + Salted(out, h, in, buf[:8]) + } + return f, nil + case 3: + _, err = io.ReadFull(r, buf[:9]) + if err != nil { + return + } + count := decodeCount(buf[8]) + f := func(out, in []byte) { + Iterated(out, h, in, buf[:8], count) + } + return f, nil + } + + return nil, errors.UnsupportedError("S2K function") +} + +// Serialize salts and stretches the given passphrase and writes the +// resulting key into key. It also serializes an S2K descriptor to +// w. The key stretching can be configured with c, which may be +// nil. In that case, sensible defaults will be used. +func Serialize(w io.Writer, key []byte, rand io.Reader, passphrase []byte, c *Config) error { + var buf [11]byte + buf[0] = 3 /* iterated and salted */ + buf[1], _ = HashToHashId(c.hash()) + salt := buf[2:10] + if _, err := io.ReadFull(rand, salt); err != nil { + return err + } + encodedCount := c.encodedCount() + count := decodeCount(encodedCount) + buf[10] = encodedCount + if _, err := w.Write(buf[:]); err != nil { + return err + } + + Iterated(key, c.hash().New(), passphrase, salt, count) + return nil +} + +// hashToHashIdMapping contains pairs relating OpenPGP's hash identifier with +// Go's crypto.Hash type. See RFC 4880, section 9.4. +var hashToHashIdMapping = []struct { + id byte + hash crypto.Hash + name string +}{ + {1, crypto.MD5, "MD5"}, + {2, crypto.SHA1, "SHA1"}, + {3, crypto.RIPEMD160, "RIPEMD160"}, + {8, crypto.SHA256, "SHA256"}, + {9, crypto.SHA384, "SHA384"}, + {10, crypto.SHA512, "SHA512"}, + {11, crypto.SHA224, "SHA224"}, +} + +// HashIdToHash returns a crypto.Hash which corresponds to the given OpenPGP +// hash id. +func HashIdToHash(id byte) (h crypto.Hash, ok bool) { + for _, m := range hashToHashIdMapping { + if m.id == id { + return m.hash, true + } + } + return 0, false +} + +// HashIdToString returns the name of the hash function corresponding to the +// given OpenPGP hash id, or panics if id is unknown. +func HashIdToString(id byte) (name string, ok bool) { + for _, m := range hashToHashIdMapping { + if m.id == id { + return m.name, true + } + } + + return "", false +} + +// HashIdToHash returns an OpenPGP hash id which corresponds the given Hash. +func HashToHashId(h crypto.Hash) (id byte, ok bool) { + for _, m := range hashToHashIdMapping { + if m.hash == h { + return m.id, true + } + } + return 0, false +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/s2k/s2k_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/s2k/s2k_test.go new file mode 100644 index 00000000000..183d26056b1 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/s2k/s2k_test.go @@ -0,0 +1,137 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package s2k + +import ( + "bytes" + "crypto" + _ "crypto/md5" + "crypto/rand" + "crypto/sha1" + _ "crypto/sha256" + _ "crypto/sha512" + "encoding/hex" + "testing" + + _ "golang.org/x/crypto/ripemd160" +) + +var saltedTests = []struct { + in, out string +}{ + {"hello", "10295ac1"}, + {"world", "ac587a5e"}, + {"foo", "4dda8077"}, + {"bar", "bd8aac6b9ea9cae04eae6a91c6133b58b5d9a61c14f355516ed9370456"}, + {"x", "f1d3f289"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "e00d7b45"}, +} + +func TestSalted(t *testing.T) { + h := sha1.New() + salt := [4]byte{1, 2, 3, 4} + + for i, test := range saltedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Salted(out, h, []byte(test.in), salt[:]) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + +var iteratedTests = []struct { + in, out string +}{ + {"hello", "83126105"}, + {"world", "6fa317f9"}, + {"foo", "8fbc35b9"}, + {"bar", "2af5a99b54f093789fd657f19bd245af7604d0f6ae06f66602a46a08ae"}, + {"x", "5a684dfe"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "18955174"}, +} + +func TestIterated(t *testing.T) { + h := sha1.New() + salt := [4]byte{4, 3, 2, 1} + + for i, test := range iteratedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Iterated(out, h, []byte(test.in), salt[:], 31) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + +var parseTests = []struct { + spec, in, out string +}{ + /* Simple with SHA1 */ + {"0002", "hello", "aaf4c61d"}, + /* Salted with SHA1 */ + {"01020102030405060708", "hello", "f4f7d67e"}, + /* Iterated with SHA1 */ + {"03020102030405060708f1", "hello", "f2a57b7c"}, +} + +func TestParse(t *testing.T) { + for i, test := range parseTests { + spec, _ := hex.DecodeString(test.spec) + buf := bytes.NewBuffer(spec) + f, err := Parse(buf) + if err != nil { + t.Errorf("%d: Parse returned error: %s", i, err) + continue + } + + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + f(out, []byte(test.in)) + if !bytes.Equal(out, expected) { + t.Errorf("%d: output got: %x want: %x", i, out, expected) + } + if testing.Short() { + break + } + } +} + +func TestSerialize(t *testing.T) { + hashes := []crypto.Hash{crypto.MD5, crypto.SHA1, crypto.RIPEMD160, + crypto.SHA256, crypto.SHA384, crypto.SHA512, crypto.SHA224} + testCounts := []int{-1, 0, 1024, 65536, 4063232, 65011712} + for _, h := range hashes { + for _, c := range testCounts { + testSerializeConfig(t, &Config{Hash: h, S2KCount: c}) + } + } +} + +func testSerializeConfig(t *testing.T, c *Config) { + t.Logf("Running testSerializeConfig() with config: %+v", c) + + buf := bytes.NewBuffer(nil) + key := make([]byte, 16) + passphrase := []byte("testing") + err := Serialize(buf, key, rand.Reader, passphrase, c) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + f, err := Parse(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + key2 := make([]byte, len(key)) + f(key2, passphrase) + if !bytes.Equal(key2, key) { + t.Errorf("keys don't match: %x (serialied) vs %x (parsed)", key, key2) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/write.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/write.go new file mode 100644 index 00000000000..e4711746840 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/write.go @@ -0,0 +1,374 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto" + "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/errors" + "golang.org/x/crypto/openpgp/packet" + "golang.org/x/crypto/openpgp/s2k" + "hash" + "io" + "strconv" + "time" +) + +// DetachSign signs message with the private key from signer (which must +// already have been decrypted) and writes the signature to w. +// If config is nil, sensible defaults will be used. +func DetachSign(w io.Writer, signer *Entity, message io.Reader, config *packet.Config) error { + return detachSign(w, signer, message, packet.SigTypeBinary, config) +} + +// ArmoredDetachSign signs message with the private key from signer (which +// must already have been decrypted) and writes an armored signature to w. +// If config is nil, sensible defaults will be used. +func ArmoredDetachSign(w io.Writer, signer *Entity, message io.Reader, config *packet.Config) (err error) { + return armoredDetachSign(w, signer, message, packet.SigTypeBinary, config) +} + +// DetachSignText signs message (after canonicalising the line endings) with +// the private key from signer (which must already have been decrypted) and +// writes the signature to w. +// If config is nil, sensible defaults will be used. +func DetachSignText(w io.Writer, signer *Entity, message io.Reader, config *packet.Config) error { + return detachSign(w, signer, message, packet.SigTypeText, config) +} + +// ArmoredDetachSignText signs message (after canonicalising the line endings) +// with the private key from signer (which must already have been decrypted) +// and writes an armored signature to w. +// If config is nil, sensible defaults will be used. +func ArmoredDetachSignText(w io.Writer, signer *Entity, message io.Reader, config *packet.Config) error { + return armoredDetachSign(w, signer, message, packet.SigTypeText, config) +} + +func armoredDetachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType, config *packet.Config) (err error) { + out, err := armor.Encode(w, SignatureType, nil) + if err != nil { + return + } + err = detachSign(out, signer, message, sigType, config) + if err != nil { + return + } + return out.Close() +} + +func detachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType, config *packet.Config) (err error) { + if signer.PrivateKey == nil { + return errors.InvalidArgumentError("signing key doesn't have a private key") + } + if signer.PrivateKey.Encrypted { + return errors.InvalidArgumentError("signing key is encrypted") + } + + sig := new(packet.Signature) + sig.SigType = sigType + sig.PubKeyAlgo = signer.PrivateKey.PubKeyAlgo + sig.Hash = config.Hash() + sig.CreationTime = config.Now() + sig.IssuerKeyId = &signer.PrivateKey.KeyId + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + io.Copy(wrappedHash, message) + + err = sig.Sign(h, signer.PrivateKey, config) + if err != nil { + return + } + + return sig.Serialize(w) +} + +// FileHints contains metadata about encrypted files. This metadata is, itself, +// encrypted. +type FileHints struct { + // IsBinary can be set to hint that the contents are binary data. + IsBinary bool + // FileName hints at the name of the file that should be written. It's + // truncated to 255 bytes if longer. It may be empty to suggest that the + // file should not be written to disk. It may be equal to "_CONSOLE" to + // suggest the data should not be written to disk. + FileName string + // ModTime contains the modification time of the file, or the zero time if not applicable. + ModTime time.Time +} + +// SymmetricallyEncrypt acts like gpg -c: it encrypts a file with a passphrase. +// The resulting WriteCloser must be closed after the contents of the file have +// been written. +// If config is nil, sensible defaults will be used. +func SymmetricallyEncrypt(ciphertext io.Writer, passphrase []byte, hints *FileHints, config *packet.Config) (plaintext io.WriteCloser, err error) { + if hints == nil { + hints = &FileHints{} + } + + key, err := packet.SerializeSymmetricKeyEncrypted(ciphertext, passphrase, config) + if err != nil { + return + } + w, err := packet.SerializeSymmetricallyEncrypted(ciphertext, config.Cipher(), key, config) + if err != nil { + return + } + + literaldata := w + if algo := config.Compression(); algo != packet.CompressionNone { + var compConfig *packet.CompressionConfig + if config != nil { + compConfig = config.CompressionConfig + } + literaldata, err = packet.SerializeCompressed(w, algo, compConfig) + if err != nil { + return + } + } + + var epochSeconds uint32 + if !hints.ModTime.IsZero() { + epochSeconds = uint32(hints.ModTime.Unix()) + } + return packet.SerializeLiteral(literaldata, hints.IsBinary, hints.FileName, epochSeconds) +} + +// intersectPreferences mutates and returns a prefix of a that contains only +// the values in the intersection of a and b. The order of a is preserved. +func intersectPreferences(a []uint8, b []uint8) (intersection []uint8) { + var j int + for _, v := range a { + for _, v2 := range b { + if v == v2 { + a[j] = v + j++ + break + } + } + } + + return a[:j] +} + +func hashToHashId(h crypto.Hash) uint8 { + v, ok := s2k.HashToHashId(h) + if !ok { + panic("tried to convert unknown hash") + } + return v +} + +// Encrypt encrypts a message to a number of recipients and, optionally, signs +// it. hints contains optional information, that is also encrypted, that aids +// the recipients in processing the message. The resulting WriteCloser must +// be closed after the contents of the file have been written. +// If config is nil, sensible defaults will be used. +func Encrypt(ciphertext io.Writer, to []*Entity, signed *Entity, hints *FileHints, config *packet.Config) (plaintext io.WriteCloser, err error) { + var signer *packet.PrivateKey + if signed != nil { + signKey, ok := signed.signingKey(config.Now()) + if !ok { + return nil, errors.InvalidArgumentError("no valid signing keys") + } + signer = signKey.PrivateKey + if signer.Encrypted { + return nil, errors.InvalidArgumentError("signing key must be decrypted") + } + } + + // These are the possible ciphers that we'll use for the message. + candidateCiphers := []uint8{ + uint8(packet.CipherAES128), + uint8(packet.CipherAES256), + uint8(packet.CipherCAST5), + } + // These are the possible hash functions that we'll use for the signature. + candidateHashes := []uint8{ + hashToHashId(crypto.SHA256), + hashToHashId(crypto.SHA512), + hashToHashId(crypto.SHA1), + hashToHashId(crypto.RIPEMD160), + } + // In the event that a recipient doesn't specify any supported ciphers + // or hash functions, these are the ones that we assume that every + // implementation supports. + defaultCiphers := candidateCiphers[len(candidateCiphers)-1:] + defaultHashes := candidateHashes[len(candidateHashes)-1:] + + encryptKeys := make([]Key, len(to)) + for i := range to { + var ok bool + encryptKeys[i], ok = to[i].encryptionKey(config.Now()) + if !ok { + return nil, errors.InvalidArgumentError("cannot encrypt a message to key id " + strconv.FormatUint(to[i].PrimaryKey.KeyId, 16) + " because it has no encryption keys") + } + + sig := to[i].primaryIdentity().SelfSignature + + preferredSymmetric := sig.PreferredSymmetric + if len(preferredSymmetric) == 0 { + preferredSymmetric = defaultCiphers + } + preferredHashes := sig.PreferredHash + if len(preferredHashes) == 0 { + preferredHashes = defaultHashes + } + candidateCiphers = intersectPreferences(candidateCiphers, preferredSymmetric) + candidateHashes = intersectPreferences(candidateHashes, preferredHashes) + } + + if len(candidateCiphers) == 0 || len(candidateHashes) == 0 { + return nil, errors.InvalidArgumentError("cannot encrypt because recipient set shares no common algorithms") + } + + cipher := packet.CipherFunction(candidateCiphers[0]) + // If the cipher specifed by config is a candidate, we'll use that. + configuredCipher := config.Cipher() + for _, c := range candidateCiphers { + cipherFunc := packet.CipherFunction(c) + if cipherFunc == configuredCipher { + cipher = cipherFunc + break + } + } + + var hash crypto.Hash + for _, hashId := range candidateHashes { + if h, ok := s2k.HashIdToHash(hashId); ok && h.Available() { + hash = h + break + } + } + + // If the hash specified by config is a candidate, we'll use that. + if configuredHash := config.Hash(); configuredHash.Available() { + for _, hashId := range candidateHashes { + if h, ok := s2k.HashIdToHash(hashId); ok && h == configuredHash { + hash = h + break + } + } + } + + if hash == 0 { + hashId := candidateHashes[0] + name, ok := s2k.HashIdToString(hashId) + if !ok { + name = "#" + strconv.Itoa(int(hashId)) + } + return nil, errors.InvalidArgumentError("cannot encrypt because no candidate hash functions are compiled in. (Wanted " + name + " in this case.)") + } + + symKey := make([]byte, cipher.KeySize()) + if _, err := io.ReadFull(config.Random(), symKey); err != nil { + return nil, err + } + + for _, key := range encryptKeys { + if err := packet.SerializeEncryptedKey(ciphertext, key.PublicKey, cipher, symKey, config); err != nil { + return nil, err + } + } + + encryptedData, err := packet.SerializeSymmetricallyEncrypted(ciphertext, cipher, symKey, config) + if err != nil { + return + } + + if signer != nil { + ops := &packet.OnePassSignature{ + SigType: packet.SigTypeBinary, + Hash: hash, + PubKeyAlgo: signer.PubKeyAlgo, + KeyId: signer.KeyId, + IsLast: true, + } + if err := ops.Serialize(encryptedData); err != nil { + return nil, err + } + } + + if hints == nil { + hints = &FileHints{} + } + + w := encryptedData + if signer != nil { + // If we need to write a signature packet after the literal + // data then we need to stop literalData from closing + // encryptedData. + w = noOpCloser{encryptedData} + + } + var epochSeconds uint32 + if !hints.ModTime.IsZero() { + epochSeconds = uint32(hints.ModTime.Unix()) + } + literalData, err := packet.SerializeLiteral(w, hints.IsBinary, hints.FileName, epochSeconds) + if err != nil { + return nil, err + } + + if signer != nil { + return signatureWriter{encryptedData, literalData, hash, hash.New(), signer, config}, nil + } + return literalData, nil +} + +// signatureWriter hashes the contents of a message while passing it along to +// literalData. When closed, it closes literalData, writes a signature packet +// to encryptedData and then also closes encryptedData. +type signatureWriter struct { + encryptedData io.WriteCloser + literalData io.WriteCloser + hashType crypto.Hash + h hash.Hash + signer *packet.PrivateKey + config *packet.Config +} + +func (s signatureWriter) Write(data []byte) (int, error) { + s.h.Write(data) + return s.literalData.Write(data) +} + +func (s signatureWriter) Close() error { + sig := &packet.Signature{ + SigType: packet.SigTypeBinary, + PubKeyAlgo: s.signer.PubKeyAlgo, + Hash: s.hashType, + CreationTime: s.config.Now(), + IssuerKeyId: &s.signer.KeyId, + } + + if err := sig.Sign(s.h, s.signer, s.config); err != nil { + return err + } + if err := s.literalData.Close(); err != nil { + return err + } + if err := sig.Serialize(s.encryptedData); err != nil { + return err + } + return s.encryptedData.Close() +} + +// noOpCloser is like an ioutil.NopCloser, but for an io.Writer. +// TODO: we have two of these in OpenPGP packages alone. This probably needs +// to be promoted somewhere more common. +type noOpCloser struct { + w io.Writer +} + +func (c noOpCloser) Write(data []byte) (n int, err error) { + return c.w.Write(data) +} + +func (c noOpCloser) Close() error { + return nil +} diff --git a/Godeps/_workspace/src/golang.org/x/crypto/openpgp/write_test.go b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/write_test.go new file mode 100644 index 00000000000..9f8c358b037 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/crypto/openpgp/write_test.go @@ -0,0 +1,234 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "io" + "io/ioutil" + "testing" + "time" +) + +func TestSignDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message, nil) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey1KeyId) +} + +func TestSignTextDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSignText(out, kring[0], message, nil) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey1KeyId) +} + +func TestSignDetachedDSA(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(dsaTestKeyPrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message, nil) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey3KeyId) +} + +func TestNewEntity(t *testing.T) { + if testing.Short() { + return + } + + e, err := NewEntity("Test User", "test", "test@example.com", nil) + if err != nil { + t.Errorf("failed to create entity: %s", err) + return + } + + w := bytes.NewBuffer(nil) + if err := e.SerializePrivate(w, nil); err != nil { + t.Errorf("failed to serialize entity: %s", err) + return + } + serialized := w.Bytes() + + el, err := ReadKeyRing(w) + if err != nil { + t.Errorf("failed to reparse entity: %s", err) + return + } + + if len(el) != 1 { + t.Errorf("wrong number of entities found, got %d, want 1", len(el)) + } + + w = bytes.NewBuffer(nil) + if err := e.SerializePrivate(w, nil); err != nil { + t.Errorf("failed to serialize entity second time: %s", err) + return + } + + if !bytes.Equal(w.Bytes(), serialized) { + t.Errorf("results differed") + } +} + +func TestSymmetricEncryption(t *testing.T) { + buf := new(bytes.Buffer) + plaintext, err := SymmetricallyEncrypt(buf, []byte("testing"), nil, nil) + if err != nil { + t.Errorf("error writing headers: %s", err) + return + } + message := []byte("hello world\n") + _, err = plaintext.Write(message) + if err != nil { + t.Errorf("error writing to plaintext writer: %s", err) + } + err = plaintext.Close() + if err != nil { + t.Errorf("error closing plaintext writer: %s", err) + } + + md, err := ReadMessage(buf, nil, func(keys []Key, symmetric bool) ([]byte, error) { + return []byte("testing"), nil + }, nil) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + messageBuf := bytes.NewBuffer(nil) + _, err = io.Copy(messageBuf, md.UnverifiedBody) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + if !bytes.Equal(message, messageBuf.Bytes()) { + t.Errorf("recovered message incorrect got '%s', want '%s'", messageBuf.Bytes(), message) + } +} + +var testEncryptionTests = []struct { + keyRingHex string + isSigned bool +}{ + { + testKeys1And2PrivateHex, + false, + }, + { + testKeys1And2PrivateHex, + true, + }, + { + dsaElGamalTestKeysHex, + false, + }, + { + dsaElGamalTestKeysHex, + true, + }, +} + +func TestEncryption(t *testing.T) { + for i, test := range testEncryptionTests { + kring, _ := ReadKeyRing(readerFromHex(test.keyRingHex)) + + passphrase := []byte("passphrase") + for _, entity := range kring { + if entity.PrivateKey != nil && entity.PrivateKey.Encrypted { + err := entity.PrivateKey.Decrypt(passphrase) + if err != nil { + t.Errorf("#%d: failed to decrypt key", i) + } + } + for _, subkey := range entity.Subkeys { + if subkey.PrivateKey != nil && subkey.PrivateKey.Encrypted { + err := subkey.PrivateKey.Decrypt(passphrase) + if err != nil { + t.Errorf("#%d: failed to decrypt subkey", i) + } + } + } + } + + var signed *Entity + if test.isSigned { + signed = kring[0] + } + + buf := new(bytes.Buffer) + w, err := Encrypt(buf, kring[:1], signed, nil /* no hints */, nil) + if err != nil { + t.Errorf("#%d: error in Encrypt: %s", i, err) + continue + } + + const message = "testing" + _, err = w.Write([]byte(message)) + if err != nil { + t.Errorf("#%d: error writing plaintext: %s", i, err) + continue + } + err = w.Close() + if err != nil { + t.Errorf("#%d: error closing WriteCloser: %s", i, err) + continue + } + + md, err := ReadMessage(buf, kring, nil /* no prompt */, nil) + if err != nil { + t.Errorf("#%d: error reading message: %s", i, err) + continue + } + + testTime, _ := time.Parse("2006-01-02", "2013-07-01") + if test.isSigned { + signKey, _ := kring[0].signingKey(testTime) + expectedKeyId := signKey.PublicKey.KeyId + if md.SignedByKeyId != expectedKeyId { + t.Errorf("#%d: message signed by wrong key id, got: %d, want: %d", i, *md.SignedBy, expectedKeyId) + } + if md.SignedBy == nil { + t.Errorf("#%d: failed to find the signing Entity", i) + } + } + + plaintext, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("#%d: error reading encrypted contents: %s", i, err) + continue + } + + encryptKey, _ := kring[0].encryptionKey(testTime) + expectedKeyId := encryptKey.PublicKey.KeyId + if len(md.EncryptedToKeyIds) != 1 || md.EncryptedToKeyIds[0] != expectedKeyId { + t.Errorf("#%d: expected message to be encrypted to %v, but got %#v", i, expectedKeyId, md.EncryptedToKeyIds) + } + + if string(plaintext) != message { + t.Errorf("#%d: got: %s, want: %s", i, string(plaintext), message) + } + + if test.isSigned { + if md.SignatureError != nil { + t.Errorf("#%d: signature error: %s", i, md.SignatureError) + } + if md.Signature == nil { + t.Error("signature missing") + } + } + } +}