System tests - get working under podman-in-podman

Skopeo CI tests run under podman; hence the registries
run in the tests will be podman-in-podman. This requires
complex muckery to make work:

 - install bats, jq, and podman in the test image
 - add new test-system Make target. It runs podman
   with /var/lib/containers bind-mounted to a tmpdir
   and with other necessary options; and invokes a
   test script that hack-edits /etc/containers/storage.conf
   before running podman for the first time.
 - add --cgroup-manager=cgroupfs option to podman
   invocations in BATS: without this, podman-in-podman
   fails with:
       systemd cgroup flag passed, but systemd support for managing cgroups is not available

Also: gpg --pinentry-mode option is not available on all
our test platforms. Check for it before using.

Signed-off-by: Ed Santiago <santiago@redhat.com>

Dockerfile

@@ -10,6 +10,7 @@ RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-go-md
 	gnupg \
 	# OpenShift deps
 	which tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \
+        bats jq podman \
 	&& dnf clean all
 
 # Install two versions of the registry. The first is an older version that

Makefile

@@ -138,12 +138,23 @@ install-completions:
 shell: build-container
 	$(CONTAINER_RUN) bash
 
-check: validate test-unit test-integration
+check: validate test-unit test-integration test-system
 
 # The tests can run out of entropy and block in containers, so replace /dev/random.
 test-integration: build-container
 	$(CONTAINER_RUN) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-integration'
 
+# complicated set of options needed to run podman-in-podman
+test-system: build-container
+	DTEMP=$(shell mktemp -d --tmpdir=/var/tmp podman-tmp.XXXXXX); \
+	$(CONTAINER_CMD) --privileged --net=host \
+	    -v $$DTEMP:/var/lib/containers:Z \
+            "$(IMAGE)" \
+            bash -c 'BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-system'; \
+	rc=$$?; \
+	$(RM) -rf $$DTEMP; \
+	exit $$rc
+
 test-unit: build-container
 	# Just call (make test unit-local) here instead of worrying about environment differences, e.g. GO15VENDOREXPERIMENT.
 	$(CONTAINER_RUN) make test-unit-local BUILDTAGS='$(BUILDTAGS)'

hack/make/test-system

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Before running podman for the first time, make sure
+# to set storage to vfs (not overlay): podman-in-podman
+# doesn't work with overlay. And, disable mountopt,
+# which causes error with vfs.
+sed -i \
+    -e 's/^driver\s*=.*/driver = "vfs"/' \
+    -e 's/^mountopt/#mountopt/' \
+    /etc/containers/storage.conf
+
+# Build skopeo, install into /usr/bin
+make binary-local ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"}
+make install
+
+# Run tests
+SKOPEO_BINARY=/usr/bin/skopeo bats --tap systemtest

systemtest/050-signing.bats

@@ -11,8 +11,17 @@ function setup() {
     # Create dummy gpg keys
     export GNUPGHOME=$TESTDIR/skopeo-gpg
     mkdir --mode=0700 $GNUPGHOME
+
+    # gpg on f30 needs this, otherwise:
+    #   gpg: agent_genkey failed: Inappropriate ioctl for device
+    # ...but gpg on f29 (and, probably, Ubuntu) doesn't grok this
+    GPGOPTS='--pinentry-mode loopback'
+    if gpg --pinentry-mode asdf 2>&1 | grep -qi 'Invalid option'; then
+        GPGOPTS=
+    fi
+
     for k in alice bob;do
-        gpg --batch --pinentry-mode loopback --gen-key --passphrase '' <<END_GPG
+        gpg --batch $GPGOPTS --gen-key --passphrase '' <<END_GPG
 Key-Type: RSA
 Name-Real: Test key - $k
 Name-email: $k@test.redhat.com

systemtest/helpers.bash

@@ -265,6 +265,10 @@ start_registry() {
 
     local -a reg_args=(-v $AUTHDIR:/auth:Z -p $port:5000)
 
+    # cgroup option necessary under podman-in-podman (CI tests),
+    # and doesn't seem to do any harm otherwise.
+    PODMAN="podman --cgroup-manager=cgroupfs"
+
     # Called with --testuser? Create an htpasswd file
     if [[ -n $testuser ]]; then
         if [[ -z $testpassword ]]; then
@@ -272,7 +276,7 @@ start_registry() {
         fi
 
         if ! egrep -q "^$testuser:" $AUTHDIR/htpasswd; then
-            podman run --rm --entrypoint htpasswd registry:2 \
+            $PODMAN run --rm --entrypoint htpasswd registry:2 \
                    -Bbn $testuser $testpassword >> $AUTHDIR/htpasswd
         fi
 
@@ -305,7 +309,7 @@ start_registry() {
         cp $CERT $TESTDIR/client-auth/
     fi
 
-    podman run -d --name $name "${reg_args[@]}" registry:2
+    $PODMAN run -d --name $name "${reg_args[@]}" registry:2
 }
 
 # END   helpers for starting/stopping registries