Add integration tests for (skopeo copy) against the Atomic Registry

This builds from the image-signatures-rest branch for
https://github.com/openshift/origin/pull/9181 .

Testing push, pull, streaming.

Does not test working with the other Docker registries built in
Dockerfile; I will leave that to the author of that code :)

Note that this relies on an internet connection for pulling from the
Docker Hub (which is incidentally tested by that); pushing to no Docker
Registry, neither local nor Hub, is tested by this.

The tests only run in a container because the (oc login) / (docker
login)-like code modifies files in a home directory; the new
SKOPEO_CONTAINER_TESTS environment variable should protect against
accidental non-container runs.

Dockerfile

@@ -40,6 +40,16 @@ RUN set -x \
 		< "$DRV1/contrib/boto_header_patch.diff" \
 	&& dnf -y update && dnf install -y m2crypto
 
+RUN set -x \
+	&& yum install -y which git tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \
+	&& export GOPATH=$(mktemp -d) \
+	# && git clone git://github.com/openshift/origin "$GOPATH/src/github.com/openshift/origin" \
+	&& git clone -b image-signatures-rest git://github.com/miminar/origin "$GOPATH/src/github.com/openshift/origin" \
+	&& (cd "$GOPATH/src/github.com/openshift/origin" && make clean build && make all WHAT=cmd/dockerregistry) \
+	&& cp -a "$GOPATH/src/github.com/openshift/origin/_output/local/bin/linux"/*/* /usr/local/bin \
+	&& cp "$GOPATH/src/github.com/openshift/origin/images/dockerregistry/config.yml" /atomic-registry-config.yml \
+	&& mkdir /registry
+
 ENV GOPATH /usr/share/gocode:/go
 ENV PATH $GOPATH/bin:/usr/share/gocode/bin:$PATH
 RUN go get github.com/golang/lint/golint

Makefile

@@ -51,7 +51,7 @@ check: validate test-unit test-integration
 
 # The tests can run out of entropy and block in containers, so replace /dev/random.
 test-integration: build-container
-	$(DOCKER_RUN_DOCKER) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; hack/make.sh test-integration'
+	$(DOCKER_RUN_DOCKER) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 hack/make.sh test-integration'
 
 test-unit: build-container
 	# Just call (make test unit-local) here instead of worrying about environment differences, e.g. GO15VENDOREXPERIMENT.

integration/copy_test.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/go-check/check"
+	"github.com/projectatomic/skopeo/docker/utils"
+)
+
+func init() {
+	check.Suite(&CopySuite{})
+}
+
+type CopySuite struct {
+	cluster *openshiftCluster
+}
+
+func (s *CopySuite) SetUpSuite(c *check.C) {
+	if os.Getenv("SKOPEO_CONTAINER_TESTS") != "1" {
+		c.Skip("Not running in a container, refusing to affect user state")
+	}
+
+	s.cluster = startOpenshiftCluster(c)
+
+	for _, stream := range []string{"unsigned"} {
+		isJSON := fmt.Sprintf(`{
+			"kind": "ImageStream",
+			"apiVersion": "v1",
+			"metadata": {
+			    "name": "%s"
+			},
+			"spec": {}
+		}`, stream)
+		runCommandWithInput(c, isJSON, "oc", "create", "-f", "-")
+	}
+}
+
+func (s *CopySuite) TearDownSuite(c *check.C) {
+	if s.cluster != nil {
+		s.cluster.tearDown()
+	}
+}
+
+// The most basic (skopeo copy) use:
+func (s *CopySuite) TestCopySimple(c *check.C) {
+	dir1, err := ioutil.TempDir("", "copy-1")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(dir1)
+	dir2, err := ioutil.TempDir("", "copy-2")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(dir2)
+
+	// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
+	// "pull": docker: → dir:
+	assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "dir:"+dir1)
+	// "push": dir: → atomic:
+	assertSkopeoSucceeds(c, "", "--debug", "copy", "dir:"+dir1, "atomic:myns/unsigned:unsigned")
+	// The result of pushing and pulling is an unmodified image.
+	assertSkopeoSucceeds(c, "", "copy", "atomic:myns/unsigned:unsigned", "dir:"+dir2)
+	out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2)
+	c.Assert(out, check.Equals, "")
+
+	// FIXME: Also check pushing to docker://
+}
+
+// Streaming (skopeo copy)
+func (s *CopySuite) TestCopyStreaming(c *check.C) {
+	dir1, err := ioutil.TempDir("", "streaming-1")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(dir1)
+	dir2, err := ioutil.TempDir("", "streaming-2")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(dir2)
+
+	// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
+	// streaming: docker: → atomic:
+	assertSkopeoSucceeds(c, "", "--debug", "copy", "docker://busybox:1-glibc", "atomic:myns/unsigned:streaming")
+	// Compare (copies of) the original and the copy:
+	assertSkopeoSucceeds(c, "", "copy", "docker://busybox:1-glibc", "dir:"+dir1)
+	assertSkopeoSucceeds(c, "", "copy", "atomic:myns/unsigned:streaming", "dir:"+dir2)
+	// The manifests will have different JWS signatures; so, compare the manifests by digests, which
+	// strips the signatures, and remove them, comparing the rest file by file.
+	digests := []string{}
+	for _, dir := range []string{dir1, dir2} {
+		manifestPath := filepath.Join(dir, "manifest.json")
+		manifest, err := ioutil.ReadFile(manifestPath)
+		c.Assert(err, check.IsNil)
+		digest, err := utils.ManifestDigest(manifest)
+		c.Assert(err, check.IsNil)
+		digests = append(digests, digest)
+		err = os.Remove(manifestPath)
+		c.Assert(err, check.IsNil)
+		c.Logf("Manifest file %s (digest %s) removed", manifestPath, digest)
+	}
+	c.Assert(digests[0], check.Equals, digests[1])
+	out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2)
+	c.Assert(out, check.Equals, "")
+	// FIXME: Also check pushing to docker://
+}

integration/openshift.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"bufio"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/pkg/homedir"
+	"github.com/go-check/check"
+)
+
+// openshiftCluster is an OpenShift API master and integrated registry
+// running on localhost.
+type openshiftCluster struct {
+	c          *check.C
+	workingDir string
+	master     *exec.Cmd
+	registry   *exec.Cmd
+}
+
+// startOpenshiftCluster creates a new openshiftCluster.
+// WARNING: This affects state in users' home directory! Only run
+// in isolated test environment.
+func startOpenshiftCluster(c *check.C) *openshiftCluster {
+	cluster := &openshiftCluster{c: c}
+
+	dir, err := ioutil.TempDir("", "openshift-cluster")
+	cluster.c.Assert(err, check.IsNil)
+	cluster.workingDir = dir
+
+	cluster.startMaster()
+	cluster.startRegistry()
+	cluster.ocLoginToProject()
+	cluster.dockerLogin()
+
+	return cluster
+}
+
+// startMaster starts the OpenShift master (etcd+API server) and waits for it to be ready, or terminates on failure.
+func (c *openshiftCluster) startMaster() {
+	c.master = exec.Command("openshift", "start", "master")
+	c.master.Dir = c.workingDir
+	stdout, err := c.master.StdoutPipe()
+	// Send both to the same pipe. This might cause the two streams to be mixed up,
+	// but logging actually goes only to stderr - this primarily ensure we log any
+	// unexpected output to stdout.
+	c.master.Stderr = c.master.Stdout
+	err = c.master.Start()
+	c.c.Assert(err, check.IsNil)
+
+	portOpen, terminatePortCheck := newPortChecker(c.c, 8443)
+	defer func() {
+		c.c.Logf("Terminating port check")
+		terminatePortCheck <- true
+	}()
+
+	terminateLogCheck := make(chan bool, 1)
+	logCheckFound := make(chan bool)
+	go func() {
+		defer func() {
+			c.c.Logf("Log checker exiting")
+		}()
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			line := scanner.Text()
+			c.c.Logf("Log line: %s", line)
+			if strings.Contains(line, "Started Origin Controllers") {
+				logCheckFound <- true
+				return
+				// FIXME? We stop reading from stdout; could this block the master?
+			}
+			// Note: we can block before we get here.
+			select {
+			case <-terminateLogCheck:
+				c.c.Logf("terminated")
+				return
+			default:
+				// Do not block here and read the next line.
+			}
+		}
+		logCheckFound <- false
+	}()
+	defer func() {
+		c.c.Logf("Terminating log check")
+		terminateLogCheck <- true
+	}()
+
+	gotPortCheck := false
+	gotLogCheck := false
+	for !gotPortCheck || !gotLogCheck {
+		c.c.Logf("Waiting for master")
+		select {
+		case <-portOpen:
+			c.c.Logf("port check done")
+			gotPortCheck = true
+		case found := <-logCheckFound:
+			c.c.Logf("log check done, found: %t", found)
+			if !found {
+				c.c.Fatal("log check done, success message not found")
+			}
+			gotLogCheck = true
+		}
+	}
+	c.c.Logf("OK, master started!")
+}
+
+// startRegistry starts the OpenShift registry and waits for it to be ready, or terminates on failure.
+func (c *openshiftCluster) startRegistry() {
+	//KUBECONFIG=openshift.local.config/master/openshift-registry.kubeconfig DOCKER_REGISTRY_URL=127.0.0.1:5000
+	c.registry = exec.Command("dockerregistry", "/atomic-registry-config.yml")
+	c.registry.Dir = c.workingDir
+	c.registry.Env = os.Environ()
+	c.registry.Env = modifyEnviron(c.registry.Env, "KUBECONFIG", "openshift.local.config/master/openshift-registry.kubeconfig")
+	c.registry.Env = modifyEnviron(c.registry.Env, "DOCKER_REGISTRY_URL", "127.0.0.1:5000")
+	consumeAndLogOutputs(c.c, "registry", c.registry)
+	err := c.registry.Start()
+	c.c.Assert(err, check.IsNil)
+
+	portOpen, terminatePortCheck := newPortChecker(c.c, 5000)
+	defer func() {
+		terminatePortCheck <- true
+	}()
+	c.c.Logf("Waiting for registry to start")
+	<-portOpen
+	c.c.Logf("OK, Registry port open")
+}
+
+// ocLogin runs (oc login) and (oc new-project) on the cluster, or terminates on failure.
+func (c *openshiftCluster) ocLoginToProject() {
+	c.c.Logf("oc login")
+	cmd := exec.Command("oc", "login", "--certificate-authority=openshift.local.config/master/ca.crt", "-u", "myuser", "-p", "mypw", "https://localhost:8443")
+	cmd.Dir = c.workingDir
+	out, err := cmd.CombinedOutput()
+	c.c.Assert(err, check.IsNil, check.Commentf("%s", out))
+	c.c.Assert(string(out), check.Matches, "(?s).*Login successful.*") // (?s) : '.' will also match newlines
+
+	outString := combinedOutputOfCommand(c.c, "oc", "new-project", "myns")
+	c.c.Assert(outString, check.Matches, `(?s).*Now using project "myns".*`) // (?s) : '.' will also match newlines
+}
+
+// dockerLogin simulates (docker login) to the cluster, or terminates on failure.
+// We do not run (docker login) directly, because that requires a running daemon and a docker package.
+func (c *openshiftCluster) dockerLogin() {
+	dockerDir := filepath.Join(homedir.Get(), ".docker")
+	err := os.Mkdir(dockerDir, 0700)
+	c.c.Assert(err, check.IsNil)
+
+	out := combinedOutputOfCommand(c.c, "oc", "config", "view", "-o", "json", "-o", "jsonpath={.users[*].user.token}")
+	c.c.Logf("oc config value: %s", out)
+	configJSON := fmt.Sprintf(`{
+		"auths": {
+			"localhost:5000": {
+				"auth": "%s",
+				"email": "unused"
+			}
+		}
+	}`, base64.StdEncoding.EncodeToString([]byte("unused:"+out)))
+	err = ioutil.WriteFile(filepath.Join(dockerDir, "config.json"), []byte(configJSON), 0600)
+	c.c.Assert(err, check.IsNil)
+}
+
+// tearDown stops the cluster services and deletes (only some!) of the state.
+func (c *openshiftCluster) tearDown() {
+	if c.registry != nil && c.registry.Process != nil {
+		c.registry.Process.Kill()
+	}
+	if c.master != nil && c.master.Process != nil {
+		c.master.Process.Kill()
+	}
+	if c.workingDir != "" {
+		os.RemoveAll(c.workingDir)
+	}
+}

integration/utils.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"io"
+	"net"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/go-check/check"
 )
@@ -38,6 +40,15 @@ func consumeAndLogOutputs(c *check.C, id string, cmd *exec.Cmd) {
 	consumeAndLogOutputStream(c, id+" stderr", stderr, err)
 }
 
+// combinedOutputOfCommand runs a command as if exec.Command().CombinedOutput(), verifies that the exit status is 0, and returns the output,
+// or terminates c on failure.
+func combinedOutputOfCommand(c *check.C, name string, args ...string) string {
+	c.Logf("Running %s %s", name, strings.Join(args, " "))
+	out, err := exec.Command(name, args...).CombinedOutput()
+	c.Assert(err, check.IsNil, check.Commentf("%s", out))
+	return string(out)
+}
+
 // assertSkopeoSucceeds runs a skopeo command as if exec.Command().CombinedOutput, verifies that the exit status is 0,
 // and optionally that the output matches a multi-line regexp if it is nonempty;
 // or terminates c on failure
@@ -77,3 +88,59 @@ func runCommandWithInput(c *check.C, input string, name string, args ...string)
 	err = cmd.Wait()
 	c.Assert(err, check.IsNil)
 }
+
+// isPortOpen returns true iff the specified port on localhost is open.
+func isPortOpen(port int) bool {
+	conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
+	if err != nil {
+		return false
+	}
+	conn.Close()
+	return true
+}
+
+// newPortChecker sets up a portOpen channel which will receive true after the specified port is open.
+// The checking can be aborted by sending a value to the terminate channel, which the caller should
+// always do using
+// defer func() {terminate <- true}()
+func newPortChecker(c *check.C, port int) (portOpen <-chan bool, terminate chan<- bool) {
+	portOpenBidi := make(chan bool)
+	// Buffered, so that sending a terminate request after the goroutine has exited does not block.
+	terminateBidi := make(chan bool, 1)
+
+	go func() {
+		defer func() {
+			c.Logf("Port checker for port %d exiting", port)
+		}()
+		for {
+			c.Logf("Checking for port %d...", port)
+			if isPortOpen(port) {
+				c.Logf("Port %d open", port)
+				portOpenBidi <- true
+				return
+			}
+			c.Logf("Sleeping for port %d", port)
+			sleepChan := time.After(100 * time.Millisecond)
+			select {
+			case <-sleepChan: // Try again
+				c.Logf("Sleeping for port %d done, will retry", port)
+			case <-terminateBidi:
+				c.Logf("Check for port %d terminated", port)
+				return
+			}
+		}
+	}()
+	return portOpenBidi, terminateBidi
+}
+
+// modifyEnviron modifies os.Environ()-like list of name=value assignments to set name to value.
+func modifyEnviron(env []string, name, value string) []string {
+	prefix := name + "="
+	res := []string{}
+	for _, e := range env {
+		if !strings.HasPrefix(e, prefix) {
+			res = append(res, e)
+		}
+	}
+	return append(res, prefix+value)
+}