From 7d379cf87adbd272dff23ef55588818c704bec88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Tue, 21 Jun 2016 17:00:18 +0200 Subject: [PATCH] Add integration tests for (skopeo copy) against the Atomic Registry This builds from the image-signatures-rest branch for https://github.com/openshift/origin/pull/9181 . Testing push, pull, streaming. Does not test working with the other Docker registries built in Dockerfile; I will leave that to the author of that code :) Note that this relies on an internet connection for pulling from the Docker Hub (which is incidentally tested by that); pushing to no Docker Registry, neither local nor Hub, is tested by this. The tests only run in a container because the (oc login) / (docker login)-like code modifies files in a home directory; the new SKOPEO_CONTAINER_TESTS environment variable should protect against accidental non-container runs. --- Dockerfile | 10 +++ Makefile | 2 +- integration/copy_test.go | 102 ++++++++++++++++++++++ integration/openshift.go | 178 +++++++++++++++++++++++++++++++++++++++ integration/utils.go | 67 +++++++++++++++ 5 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 integration/copy_test.go create mode 100644 integration/openshift.go diff --git a/Dockerfile b/Dockerfile index 33077e35..c87cecf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,16 @@ RUN set -x \ < "$DRV1/contrib/boto_header_patch.diff" \ && dnf -y update && dnf install -y m2crypto +RUN set -x \ + && yum install -y which git tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \ + && export GOPATH=$(mktemp -d) \ + # && git clone git://github.com/openshift/origin "$GOPATH/src/github.com/openshift/origin" \ + && git clone -b image-signatures-rest git://github.com/miminar/origin "$GOPATH/src/github.com/openshift/origin" \ + && (cd "$GOPATH/src/github.com/openshift/origin" && make clean build && make all WHAT=cmd/dockerregistry) \ + && cp -a "$GOPATH/src/github.com/openshift/origin/_output/local/bin/linux"/*/* /usr/local/bin \ + && cp "$GOPATH/src/github.com/openshift/origin/images/dockerregistry/config.yml" /atomic-registry-config.yml \ + && mkdir /registry + ENV GOPATH /usr/share/gocode:/go ENV PATH $GOPATH/bin:/usr/share/gocode/bin:$PATH RUN go get github.com/golang/lint/golint diff --git a/Makefile b/Makefile index 63ed005e..1c53fc1a 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ check: validate test-unit test-integration # The tests can run out of entropy and block in containers, so replace /dev/random. test-integration: build-container - $(DOCKER_RUN_DOCKER) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; hack/make.sh test-integration' + $(DOCKER_RUN_DOCKER) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 hack/make.sh test-integration' test-unit: build-container # Just call (make test unit-local) here instead of worrying about environment differences, e.g. GO15VENDOREXPERIMENT. diff --git a/integration/copy_test.go b/integration/copy_test.go new file mode 100644 index 00000000..8b0539d5 --- /dev/null +++ b/integration/copy_test.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/go-check/check" + "github.com/projectatomic/skopeo/docker/utils" +) + +func init() { + check.Suite(&CopySuite{}) +} + +type CopySuite struct { + cluster *openshiftCluster +} + +func (s *CopySuite) SetUpSuite(c *check.C) { + if os.Getenv("SKOPEO_CONTAINER_TESTS") != "1" { + c.Skip("Not running in a container, refusing to affect user state") + } + + s.cluster = startOpenshiftCluster(c) + + for _, stream := range []string{"unsigned"} { + isJSON := fmt.Sprintf(`{ + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "%s" + }, + "spec": {} + }`, stream) + runCommandWithInput(c, isJSON, "oc", "create", "-f", "-") + } +} + +func (s *CopySuite) TearDownSuite(c *check.C) { + if s.cluster != nil { + s.cluster.tearDown() + } +} + +// The most basic (skopeo copy) use: +func (s *CopySuite) TestCopySimple(c *check.C) { + dir1, err := ioutil.TempDir("", "copy-1") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir1) + dir2, err := ioutil.TempDir("", "copy-2") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir2) + + // FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection. + // "pull": docker: → dir: + assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "dir:"+dir1) + // "push": dir: → atomic: + assertSkopeoSucceeds(c, "", "--debug", "copy", "dir:"+dir1, "atomic:myns/unsigned:unsigned") + // The result of pushing and pulling is an unmodified image. + assertSkopeoSucceeds(c, "", "copy", "atomic:myns/unsigned:unsigned", "dir:"+dir2) + out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2) + c.Assert(out, check.Equals, "") + + // FIXME: Also check pushing to docker:// +} + +// Streaming (skopeo copy) +func (s *CopySuite) TestCopyStreaming(c *check.C) { + dir1, err := ioutil.TempDir("", "streaming-1") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir1) + dir2, err := ioutil.TempDir("", "streaming-2") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir2) + + // FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection. + // streaming: docker: → atomic: + assertSkopeoSucceeds(c, "", "--debug", "copy", "docker://busybox:1-glibc", "atomic:myns/unsigned:streaming") + // Compare (copies of) the original and the copy: + assertSkopeoSucceeds(c, "", "copy", "docker://busybox:1-glibc", "dir:"+dir1) + assertSkopeoSucceeds(c, "", "copy", "atomic:myns/unsigned:streaming", "dir:"+dir2) + // The manifests will have different JWS signatures; so, compare the manifests by digests, which + // strips the signatures, and remove them, comparing the rest file by file. + digests := []string{} + for _, dir := range []string{dir1, dir2} { + manifestPath := filepath.Join(dir, "manifest.json") + manifest, err := ioutil.ReadFile(manifestPath) + c.Assert(err, check.IsNil) + digest, err := utils.ManifestDigest(manifest) + c.Assert(err, check.IsNil) + digests = append(digests, digest) + err = os.Remove(manifestPath) + c.Assert(err, check.IsNil) + c.Logf("Manifest file %s (digest %s) removed", manifestPath, digest) + } + c.Assert(digests[0], check.Equals, digests[1]) + out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2) + c.Assert(out, check.Equals, "") + // FIXME: Also check pushing to docker:// +} diff --git a/integration/openshift.go b/integration/openshift.go new file mode 100644 index 00000000..e55cfa4d --- /dev/null +++ b/integration/openshift.go @@ -0,0 +1,178 @@ +package main + +import ( + "bufio" + "encoding/base64" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/docker/docker/pkg/homedir" + "github.com/go-check/check" +) + +// openshiftCluster is an OpenShift API master and integrated registry +// running on localhost. +type openshiftCluster struct { + c *check.C + workingDir string + master *exec.Cmd + registry *exec.Cmd +} + +// startOpenshiftCluster creates a new openshiftCluster. +// WARNING: This affects state in users' home directory! Only run +// in isolated test environment. +func startOpenshiftCluster(c *check.C) *openshiftCluster { + cluster := &openshiftCluster{c: c} + + dir, err := ioutil.TempDir("", "openshift-cluster") + cluster.c.Assert(err, check.IsNil) + cluster.workingDir = dir + + cluster.startMaster() + cluster.startRegistry() + cluster.ocLoginToProject() + cluster.dockerLogin() + + return cluster +} + +// startMaster starts the OpenShift master (etcd+API server) and waits for it to be ready, or terminates on failure. +func (c *openshiftCluster) startMaster() { + c.master = exec.Command("openshift", "start", "master") + c.master.Dir = c.workingDir + stdout, err := c.master.StdoutPipe() + // Send both to the same pipe. This might cause the two streams to be mixed up, + // but logging actually goes only to stderr - this primarily ensure we log any + // unexpected output to stdout. + c.master.Stderr = c.master.Stdout + err = c.master.Start() + c.c.Assert(err, check.IsNil) + + portOpen, terminatePortCheck := newPortChecker(c.c, 8443) + defer func() { + c.c.Logf("Terminating port check") + terminatePortCheck <- true + }() + + terminateLogCheck := make(chan bool, 1) + logCheckFound := make(chan bool) + go func() { + defer func() { + c.c.Logf("Log checker exiting") + }() + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + c.c.Logf("Log line: %s", line) + if strings.Contains(line, "Started Origin Controllers") { + logCheckFound <- true + return + // FIXME? We stop reading from stdout; could this block the master? + } + // Note: we can block before we get here. + select { + case <-terminateLogCheck: + c.c.Logf("terminated") + return + default: + // Do not block here and read the next line. + } + } + logCheckFound <- false + }() + defer func() { + c.c.Logf("Terminating log check") + terminateLogCheck <- true + }() + + gotPortCheck := false + gotLogCheck := false + for !gotPortCheck || !gotLogCheck { + c.c.Logf("Waiting for master") + select { + case <-portOpen: + c.c.Logf("port check done") + gotPortCheck = true + case found := <-logCheckFound: + c.c.Logf("log check done, found: %t", found) + if !found { + c.c.Fatal("log check done, success message not found") + } + gotLogCheck = true + } + } + c.c.Logf("OK, master started!") +} + +// startRegistry starts the OpenShift registry and waits for it to be ready, or terminates on failure. +func (c *openshiftCluster) startRegistry() { + //KUBECONFIG=openshift.local.config/master/openshift-registry.kubeconfig DOCKER_REGISTRY_URL=127.0.0.1:5000 + c.registry = exec.Command("dockerregistry", "/atomic-registry-config.yml") + c.registry.Dir = c.workingDir + c.registry.Env = os.Environ() + c.registry.Env = modifyEnviron(c.registry.Env, "KUBECONFIG", "openshift.local.config/master/openshift-registry.kubeconfig") + c.registry.Env = modifyEnviron(c.registry.Env, "DOCKER_REGISTRY_URL", "127.0.0.1:5000") + consumeAndLogOutputs(c.c, "registry", c.registry) + err := c.registry.Start() + c.c.Assert(err, check.IsNil) + + portOpen, terminatePortCheck := newPortChecker(c.c, 5000) + defer func() { + terminatePortCheck <- true + }() + c.c.Logf("Waiting for registry to start") + <-portOpen + c.c.Logf("OK, Registry port open") +} + +// ocLogin runs (oc login) and (oc new-project) on the cluster, or terminates on failure. +func (c *openshiftCluster) ocLoginToProject() { + c.c.Logf("oc login") + cmd := exec.Command("oc", "login", "--certificate-authority=openshift.local.config/master/ca.crt", "-u", "myuser", "-p", "mypw", "https://localhost:8443") + cmd.Dir = c.workingDir + out, err := cmd.CombinedOutput() + c.c.Assert(err, check.IsNil, check.Commentf("%s", out)) + c.c.Assert(string(out), check.Matches, "(?s).*Login successful.*") // (?s) : '.' will also match newlines + + outString := combinedOutputOfCommand(c.c, "oc", "new-project", "myns") + c.c.Assert(outString, check.Matches, `(?s).*Now using project "myns".*`) // (?s) : '.' will also match newlines +} + +// dockerLogin simulates (docker login) to the cluster, or terminates on failure. +// We do not run (docker login) directly, because that requires a running daemon and a docker package. +func (c *openshiftCluster) dockerLogin() { + dockerDir := filepath.Join(homedir.Get(), ".docker") + err := os.Mkdir(dockerDir, 0700) + c.c.Assert(err, check.IsNil) + + out := combinedOutputOfCommand(c.c, "oc", "config", "view", "-o", "json", "-o", "jsonpath={.users[*].user.token}") + c.c.Logf("oc config value: %s", out) + configJSON := fmt.Sprintf(`{ + "auths": { + "localhost:5000": { + "auth": "%s", + "email": "unused" + } + } + }`, base64.StdEncoding.EncodeToString([]byte("unused:"+out))) + err = ioutil.WriteFile(filepath.Join(dockerDir, "config.json"), []byte(configJSON), 0600) + c.c.Assert(err, check.IsNil) +} + +// tearDown stops the cluster services and deletes (only some!) of the state. +func (c *openshiftCluster) tearDown() { + if c.registry != nil && c.registry.Process != nil { + c.registry.Process.Kill() + } + if c.master != nil && c.master.Process != nil { + c.master.Process.Kill() + } + if c.workingDir != "" { + os.RemoveAll(c.workingDir) + } +} diff --git a/integration/utils.go b/integration/utils.go index 3f959db9..a6b85111 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -2,8 +2,10 @@ package main import ( "io" + "net" "os/exec" "strings" + "time" "github.com/go-check/check" ) @@ -38,6 +40,15 @@ func consumeAndLogOutputs(c *check.C, id string, cmd *exec.Cmd) { consumeAndLogOutputStream(c, id+" stderr", stderr, err) } +// combinedOutputOfCommand runs a command as if exec.Command().CombinedOutput(), verifies that the exit status is 0, and returns the output, +// or terminates c on failure. +func combinedOutputOfCommand(c *check.C, name string, args ...string) string { + c.Logf("Running %s %s", name, strings.Join(args, " ")) + out, err := exec.Command(name, args...).CombinedOutput() + c.Assert(err, check.IsNil, check.Commentf("%s", out)) + return string(out) +} + // assertSkopeoSucceeds runs a skopeo command as if exec.Command().CombinedOutput, verifies that the exit status is 0, // and optionally that the output matches a multi-line regexp if it is nonempty; // or terminates c on failure @@ -77,3 +88,59 @@ func runCommandWithInput(c *check.C, input string, name string, args ...string) err = cmd.Wait() c.Assert(err, check.IsNil) } + +// isPortOpen returns true iff the specified port on localhost is open. +func isPortOpen(port int) bool { + conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}) + if err != nil { + return false + } + conn.Close() + return true +} + +// newPortChecker sets up a portOpen channel which will receive true after the specified port is open. +// The checking can be aborted by sending a value to the terminate channel, which the caller should +// always do using +// defer func() {terminate <- true}() +func newPortChecker(c *check.C, port int) (portOpen <-chan bool, terminate chan<- bool) { + portOpenBidi := make(chan bool) + // Buffered, so that sending a terminate request after the goroutine has exited does not block. + terminateBidi := make(chan bool, 1) + + go func() { + defer func() { + c.Logf("Port checker for port %d exiting", port) + }() + for { + c.Logf("Checking for port %d...", port) + if isPortOpen(port) { + c.Logf("Port %d open", port) + portOpenBidi <- true + return + } + c.Logf("Sleeping for port %d", port) + sleepChan := time.After(100 * time.Millisecond) + select { + case <-sleepChan: // Try again + c.Logf("Sleeping for port %d done, will retry", port) + case <-terminateBidi: + c.Logf("Check for port %d terminated", port) + return + } + } + }() + return portOpenBidi, terminateBidi +} + +// modifyEnviron modifies os.Environ()-like list of name=value assignments to set name to value. +func modifyEnviron(env []string, name, value string) []string { + prefix := name + "=" + res := []string{} + for _, e := range env { + if !strings.HasPrefix(e, prefix) { + res = append(res, e) + } + } + return append(res, prefix+value) +}