From 12f0e2451901a721be2a01d6b1266e3a0e5fc567 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Tue, 14 May 2019 10:30:16 -0600 Subject: [PATCH 1/3] systemtest - new set of BATS tests for RHEL8 gating Signed-off-by: Ed Santiago --- systemtest/001-basic.bats | 19 ++ systemtest/010-inspect.bats | 67 +++++ systemtest/020-copy.bats | 79 ++++++ systemtest/030-local-registry-tls.bats | 41 +++ systemtest/040-local-registry-auth.bats | 76 ++++++ systemtest/050-signing.bats | 137 ++++++++++ systemtest/helpers.bash | 318 ++++++++++++++++++++++++ systemtest/run-tests | 19 ++ 8 files changed, 756 insertions(+) create mode 100644 systemtest/001-basic.bats create mode 100644 systemtest/010-inspect.bats create mode 100644 systemtest/020-copy.bats create mode 100644 systemtest/030-local-registry-tls.bats create mode 100644 systemtest/040-local-registry-auth.bats create mode 100644 systemtest/050-signing.bats create mode 100644 systemtest/helpers.bash create mode 100755 systemtest/run-tests diff --git a/systemtest/001-basic.bats b/systemtest/001-basic.bats new file mode 100644 index 00000000..314052fc --- /dev/null +++ b/systemtest/001-basic.bats @@ -0,0 +1,19 @@ +#!/usr/bin/env bats +# +# Simplest set of skopeo tests. If any of these fail, we have serious problems. +# + +load helpers + +# Override standard setup! We don't yet trust anything +function setup() { + : +} + +@test "skopeo version emits reasonable output" { + run_skopeo --version + + expect_output --substring "skopeo version [0-9.]+" +} + +# vim: filetype=sh diff --git a/systemtest/010-inspect.bats b/systemtest/010-inspect.bats new file mode 100644 index 00000000..ee01d123 --- /dev/null +++ b/systemtest/010-inspect.bats @@ -0,0 +1,67 @@ +#!/usr/bin/env bats +# +# Simplest test for skopeo inspect +# + +load helpers + +@test "inspect: basic" { + workdir=$TESTDIR/inspect + + remote_image=docker://quay.io/libpod/alpine_labels:latest + # Inspect remote source, then pull it. There's a small race condition + # in which the remote image can get updated between the inspect and + # the copy; let's just not worry about it. + run_skopeo inspect $remote_image + inspect_remote=$output + + # Now pull it into a directory + run_skopeo copy $remote_image dir:$workdir + expect_output --substring "Getting image source signatures" + expect_output --substring "Writing manifest to image destination" + + # Unpacked contents must include a manifest and version + [ -e $workdir/manifest.json ] + [ -e $workdir/version ] + + # Now run inspect locally + run_skopeo inspect dir:$workdir + inspect_local=$output + + # Each SHA-named file must be listed in the output of 'inspect' + for sha in $(find $workdir -type f | xargs -l1 basename | egrep '^[0-9a-f]{64}$'); do + expect_output --from="$inspect_local" --substring "sha256:$sha" \ + "Locally-extracted SHA file is present in 'inspect'" + done + + # Simple sanity check on 'inspect' output. + # For each of the given keys (LHS of the table below): + # 1) Get local and remote values + # 2) Sanity-check local value using simple expression + # 3) Confirm that local and remote values match. + # + # The reason for (2) is to make sure that we don't compare bad results + # + # The reason for a hardcoded list, instead of 'jq keys', is that RepoTags + # is always empty locally, but a list remotely. + while read key expect; do + local=$(echo "$inspect_local" | jq -r ".$key") + remote=$(echo "$inspect_remote" | jq -r ".$key") + + expect_output --from="$local" --substring "$expect" \ + "local $key is sane" + + expect_output --from="$remote" "$local" \ + "local $key matches remote" + done <$GNUPGHOME/pubkey-$k.gpg + done + + # Registries. The important part here seems to be sigstore, + # because (I guess?) the registry itself has no mechanism + # for storing or validating signatures. + REGISTRIES_D=$TESTDIR/registries.d + mkdir $REGISTRIES_D $TESTDIR/sigstore + cat >$REGISTRIES_D/registries.yaml <$POLICY_JSON </dev/null; then + echo "*** TIMED OUT ***" + false + fi + + if [ -n "$expected_rc" ]; then + if [ "$status" -ne "$expected_rc" ]; then + die "exit code is $status; expected $expected_rc" + fi + fi +} + +######### +# die # Abort with helpful message +######### +function die() { + echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2 + echo "#| FAIL: $*" >&2 + echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2 + false +} + +################### +# expect_output # Compare actual vs expected string; fail if mismatch +################### +# +# Compares $output against the given string argument. Optional second +# argument is descriptive text to show as the error message (default: +# the command most recently run by 'run_skopeo'). This text can be +# useful to isolate a failure when there are multiple identical +# run_skopeo invocations, and the difference is solely in the +# config or setup; see, e.g., run.bats:run-cmd(). +# +# By default we run an exact string comparison; use --substring to +# look for the given string anywhere in $output. +# +# By default we look in "$output", which is set in run_skopeo(). +# To override, use --from="some-other-string" (e.g. "${lines[0]}") +# +# Examples: +# +# expect_output "this is exactly what we expect" +# expect_output "foo=bar" "description of this particular test" +# expect_output --from="${lines[0]}" "expected first line" +# +function expect_output() { + # By default we examine $output, the result of run_skopeo + local actual="$output" + local check_substring= + + # option processing: recognize --from="...", --substring + local opt + for opt; do + local value=$(expr "$opt" : '[^=]*=\(.*\)') + case "$opt" in + --from=*) actual="$value"; shift;; + --substring) check_substring=1; shift;; + --) shift; break;; + -*) die "Invalid option '$opt'" ;; + *) break;; + esac + done + + local expect="$1" + local testname="${2:-${MOST_RECENT_SKOPEO_COMMAND:-[no test name given]}}" + + if [ -z "$expect" ]; then + if [ -z "$actual" ]; then + return + fi + expect='[no output]' + elif [ "$actual" = "$expect" ]; then + return + elif [ -n "$check_substring" ]; then + if [[ "$actual" =~ $expect ]]; then + return + fi + fi + + # This is a multi-line message, which may in turn contain multi-line + # output, so let's format it ourself, readably + local -a actual_split + readarray -t actual_split <<<"$actual" + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| expected: '%s'\n" "$expect" >&2 + printf "#| actual: '%s'\n" "${actual_split[0]}" >&2 + local line + for line in "${actual_split[@]:1}"; do + printf "#| > '%s'\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 + false +} + +####################### +# expect_line_count # Check the expected number of output lines +####################### +# +# ...from the most recent run_skopeo command +# +function expect_line_count() { + local expect="$1" + local testname="${2:-${MOST_RECENT_SKOPEO_COMMAND:-[no test name given]}}" + + local actual="${#lines[@]}" + if [ "$actual" -eq "$expect" ]; then + return + fi + + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| Expected %d lines of output, got %d\n" $expect $actual >&2 + printf "#| Output was:\n" >&2 + local line + for line in "${lines[@]}"; do + printf "#| >%s\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 + false +} + +# END standard helpers for running skopeo and testing results +############################################################################### +# BEGIN helpers for starting/stopping registries + +start_registry() { + local port=5000 + local testuser= + local testpassword= + local create_cert= + + # option processing: recognize --auth + local opt + for opt; do + local value=$(expr "$opt" : '[^=]*=\(.*\)') + case "$opt" in + --port=*) port="$value"; shift;; + --testuser=*) testuser="$value"; shift;; + --testpassword=*) testpassword="$value"; shift;; + --with-cert) create_cert=1; shift;; + -*) die "Invalid option '$opt'" ;; + *) break;; + esac + done + + local name=${1?start_registry() invoked without a NAME} + + # Temp directory must be defined and must exist + [[ -n $TESTDIR && -d $TESTDIR ]] + + AUTHDIR=$TESTDIR/auth + mkdir -p $AUTHDIR + + local -a reg_args=(-v $AUTHDIR:/auth:Z -p $port:5000) + + # Called with --testuser? Create an htpasswd file + if [[ -n $testuser ]]; then + if [[ -z $testpassword ]]; then + die "start_registry() invoked with testuser but no testpassword" + fi + + if ! egrep -q "^$testuser:" $AUTHDIR/htpasswd; then + podman run --rm --entrypoint htpasswd registry:2 \ + -Bbn $testuser $testpassword >> $AUTHDIR/htpasswd + fi + + reg_args+=( + -e REGISTRY_AUTH=htpasswd + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd + -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" + ) + fi + + # Called with --with-cert? Create certificates. + if [[ -n $create_cert ]]; then + CERT=$AUTHDIR/domain.cert + if [ ! -e $CERT ]; then + openssl req -newkey rsa:4096 -nodes -sha256 \ + -keyout $AUTHDIR/domain.key -x509 -days 2 \ + -out $CERT \ + -subj "/C=US/ST=Foo/L=Bar/O=Red Hat, Inc./CN=localhost" + fi + + reg_args+=( + -e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.cert + -e REGISTRY_HTTP_TLS_KEY=/auth/domain.key + ) + fi + + podman run -d --name $name "${reg_args[@]}" registry:2 +} + + +stop_registries() { + if [[ -z $SKOPEO_DEBUG_REGISTRIES ]]; then + podman rm -a -f + + if [[ -n $AUTHDIR ]]; then + rm -rf $AUTHDIR + fi + fi +} + +# END helpers for starting/stopping registries +############################################################################### +# BEGIN miscellaneous tools + +################### +# random_string # Returns a pseudorandom human-readable string +################### +# +# Numeric argument, if present, is desired length of string +# +function random_string() { + local length=${1:-10} + + head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length +} + +# END miscellaneous tools +############################################################################### diff --git a/systemtest/run-tests b/systemtest/run-tests new file mode 100755 index 00000000..004c5f56 --- /dev/null +++ b/systemtest/run-tests @@ -0,0 +1,19 @@ +#!/bin/bash +# +# run-tests - simple wrapper allowing shortcuts on invocation +# + +# FIXME +export SKOPEO_BINARY=${SKOPEO_BINARY:-/usr/bin/skopeo} + +TEST_DIR=$(dirname $0) +TESTS=$TEST_DIR + +for i; do + case "$i" in + *.bats) TESTS=$i ;; + *) TESTS=$(echo $TEST_DIR/*$i*.bats) ;; + esac +done + +bats $TESTS From 5dd3b2bffdca3f9275a1aa40fc13077ba2ef0fe5 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Mon, 20 May 2019 14:28:46 -0600 Subject: [PATCH 2/3] fixup! Incorporate review feedback from mtrmac - Got TLS registry working, and test enabled. The trick was to copy the .crt file to a separate directory *without* the .key - auth test - set up a private XDG_RUNTIME_DIR, in case tests are being run by a real user. - signing test - remove FIXME comments; questions answered. - helpers.bash - document start_registries(); save a .crt file, not .cert; and remove unused stop_registries() - it's too hard to do right, and very easy for individual tests to 'podman rm -f' - run-tests - remove SKOPEO_BINARY definition, it's inconsistent with the one in helpers.bash Signed-off-by: Ed Santiago --- systemtest/030-local-registry-tls.bats | 27 ++++++------------ systemtest/040-local-registry-auth.bats | 10 ++++--- systemtest/050-signing.bats | 4 --- systemtest/helpers.bash | 38 ++++++++++++++++--------- systemtest/run-tests | 3 -- 5 files changed, 39 insertions(+), 43 deletions(-) diff --git a/systemtest/030-local-registry-tls.bats b/systemtest/030-local-registry-tls.bats index 6c14ba86..69270199 100644 --- a/systemtest/030-local-registry-tls.bats +++ b/systemtest/030-local-registry-tls.bats @@ -1,16 +1,7 @@ #!/usr/bin/env bats # -# This is probably a never-mind. -# -# The idea is to set up a local registry with locally generated certs, -# using --dest-cert-dir to tell skopeo how to check. But no, it fails with -# -# x509: certificate signed by unknown authority -# -# Perhaps I'm missing something? Maybe I need to add something into -# /etc/pki/somewhere? If this is truly not possible to test without -# a real signature, then let's just delete this test. -# +# Confirm that skopeo will push to and pull from a local +# registry with locally-created TLS certificates. # load helpers @@ -21,15 +12,15 @@ function setup() { } @test "local registry, with cert" { - skip "doesn't work as expected" - - local remote_image=docker://busybox:latest - local localimg=docker://localhost:5000/busybox:unsigned - - # Fails with: x509: certificate signed by unknown authority - run_skopeo --debug copy --dest-cert-dir=$TESTDIR/auth \ + # Push to local registry... + run_skopeo copy --dest-cert-dir=$TESTDIR/client-auth \ docker://busybox:latest \ docker://localhost:5000/busybox:unsigned + + # ...and pull it back out + run_skopeo copy --src-cert-dir=$TESTDIR/client-auth \ + docker://localhost:5000/busybox:unsigned \ + dir:$TESTDIR/extracted } teardown() { diff --git a/systemtest/040-local-registry-auth.bats b/systemtest/040-local-registry-auth.bats index 9996ec0c..dab3bd18 100644 --- a/systemtest/040-local-registry-auth.bats +++ b/systemtest/040-local-registry-auth.bats @@ -9,8 +9,10 @@ function setup() { standard_setup # Remove old/stale cred file - _cred_file=${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containers/auth.json - rm -f $_cred_file + _cred_dir=$TESTDIR/credentials + export XDG_RUNTIME_DIR=$_cred_dir + mkdir -p $_cred_dir/containers + rm -f $_cred_dir/containers/auth.json # Start authenticated registry with random password testuser=testuser @@ -66,8 +68,8 @@ function setup() { teardown() { podman rm -f reg - if [[ -n $_cred_file ]]; then - rm -f $_cred_file + if [[ -n $_cred_dir ]]; then + rm -rf $_cred_dir fi standard_teardown diff --git a/systemtest/050-signing.bats b/systemtest/050-signing.bats index 5de7f876..7cdfc598 100644 --- a/systemtest/050-signing.bats +++ b/systemtest/050-signing.bats @@ -95,9 +95,6 @@ END_POLICY_JSON /myns/carol:latest - # No signature /open/forall:latest - # No signature, but none needed END_PUSH - # FIXME: there doesn't seem to be a way to push an image - # such as '/bob:signed', signed by bob, at the same time - # that we have :signedbyalice # Done pushing. Now try to fetch. From here on we use the --policy option. # The table below lists the paths to fetch, and the expected errors (or @@ -125,7 +122,6 @@ END_PUSH /myns/carol:latest Running image docker://localhost:5000/myns/carol:latest is rejected by policy. /open/forall:latest END_TESTS - # FIXME: why does the message for alice:unsigned say ':signed' ? } teardown() { diff --git a/systemtest/helpers.bash b/systemtest/helpers.bash index 0d4c768f..b7479138 100644 --- a/systemtest/helpers.bash +++ b/systemtest/helpers.bash @@ -220,13 +220,28 @@ function expect_line_count() { ############################################################################### # BEGIN helpers for starting/stopping registries +#################### +# start_registry # Run a local registry container +#################### +# +# Usage: start_registry [OPTIONS] NAME +# +# OPTIONS +# --port=NNNN Port to listen on (default: 5000) +# --testuser=XXX Require authentication; this is the username +# --testpassword=XXX ...and the password (these two go together) +# --with-cert Create a cert for running with TLS (not working) +# +# NAME is the container name to assign. +# start_registry() { local port=5000 local testuser= local testpassword= local create_cert= - # option processing: recognize --auth + # option processing: recognize options for running the registry + # in different modes. local opt for opt; do local value=$(expr "$opt" : '[^=]*=\(.*\)') @@ -270,7 +285,7 @@ start_registry() { # Called with --with-cert? Create certificates. if [[ -n $create_cert ]]; then - CERT=$AUTHDIR/domain.cert + CERT=$AUTHDIR/domain.crt if [ ! -e $CERT ]; then openssl req -newkey rsa:4096 -nodes -sha256 \ -keyout $AUTHDIR/domain.key -x509 -days 2 \ @@ -279,25 +294,20 @@ start_registry() { fi reg_args+=( - -e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.cert + -e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.crt -e REGISTRY_HTTP_TLS_KEY=/auth/domain.key ) + + # Copy .crt file to a directory *without* the .key one, so we can + # test the client. (If client sees a matching .key file, it fails) + # Thanks to Miloslav Trmac for this hint. + mkdir -p $TESTDIR/client-auth + cp $CERT $TESTDIR/client-auth/ fi podman run -d --name $name "${reg_args[@]}" registry:2 } - -stop_registries() { - if [[ -z $SKOPEO_DEBUG_REGISTRIES ]]; then - podman rm -a -f - - if [[ -n $AUTHDIR ]]; then - rm -rf $AUTHDIR - fi - fi -} - # END helpers for starting/stopping registries ############################################################################### # BEGIN miscellaneous tools diff --git a/systemtest/run-tests b/systemtest/run-tests index 004c5f56..21763494 100755 --- a/systemtest/run-tests +++ b/systemtest/run-tests @@ -3,9 +3,6 @@ # run-tests - simple wrapper allowing shortcuts on invocation # -# FIXME -export SKOPEO_BINARY=${SKOPEO_BINARY:-/usr/bin/skopeo} - TEST_DIR=$(dirname $0) TESTS=$TEST_DIR From 47e7cda4e971be73a08d4c88792a3fc7d817ad57 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Tue, 28 May 2019 09:11:25 -0600 Subject: [PATCH 3/3] System tests - get working under podman-in-podman Skopeo CI tests run under podman; hence the registries run in the tests will be podman-in-podman. This requires complex muckery to make work: - install bats, jq, and podman in the test image - add new test-system Make target. It runs podman with /var/lib/containers bind-mounted to a tmpdir and with other necessary options; and invokes a test script that hack-edits /etc/containers/storage.conf before running podman for the first time. - add --cgroup-manager=cgroupfs option to podman invocations in BATS: without this, podman-in-podman fails with: systemd cgroup flag passed, but systemd support for managing cgroups is not available Also: gpg --pinentry-mode option is not available on all our test platforms. Check for it before using. Signed-off-by: Ed Santiago --- Dockerfile | 1 + Makefile | 13 ++++++++++++- hack/make/test-system | 18 ++++++++++++++++++ systemtest/050-signing.bats | 11 ++++++++++- systemtest/helpers.bash | 8 ++++++-- 5 files changed, 47 insertions(+), 4 deletions(-) create mode 100755 hack/make/test-system diff --git a/Dockerfile b/Dockerfile index 3cf2be4f..28d82980 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-go-md gnupg \ # OpenShift deps which tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \ + bats jq podman \ && dnf clean all # Install two versions of the registry. The first is an older version that diff --git a/Makefile b/Makefile index 8f431701..6e5df788 100644 --- a/Makefile +++ b/Makefile @@ -138,12 +138,23 @@ install-completions: shell: build-container $(CONTAINER_RUN) bash -check: validate test-unit test-integration +check: validate test-unit test-integration test-system # The tests can run out of entropy and block in containers, so replace /dev/random. test-integration: build-container $(CONTAINER_RUN) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-integration' +# complicated set of options needed to run podman-in-podman +test-system: build-container + DTEMP=$(shell mktemp -d --tmpdir=/var/tmp podman-tmp.XXXXXX); \ + $(CONTAINER_CMD) --privileged --net=host \ + -v $$DTEMP:/var/lib/containers:Z \ + "$(IMAGE)" \ + bash -c 'BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-system'; \ + rc=$$?; \ + $(RM) -rf $$DTEMP; \ + exit $$rc + test-unit: build-container # Just call (make test unit-local) here instead of worrying about environment differences, e.g. GO15VENDOREXPERIMENT. $(CONTAINER_RUN) make test-unit-local BUILDTAGS='$(BUILDTAGS)' diff --git a/hack/make/test-system b/hack/make/test-system new file mode 100755 index 00000000..bc881e17 --- /dev/null +++ b/hack/make/test-system @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Before running podman for the first time, make sure +# to set storage to vfs (not overlay): podman-in-podman +# doesn't work with overlay. And, disable mountopt, +# which causes error with vfs. +sed -i \ + -e 's/^driver\s*=.*/driver = "vfs"/' \ + -e 's/^mountopt/#mountopt/' \ + /etc/containers/storage.conf + +# Build skopeo, install into /usr/bin +make binary-local ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"} +make install + +# Run tests +SKOPEO_BINARY=/usr/bin/skopeo bats --tap systemtest diff --git a/systemtest/050-signing.bats b/systemtest/050-signing.bats index 7cdfc598..af7e978f 100644 --- a/systemtest/050-signing.bats +++ b/systemtest/050-signing.bats @@ -11,8 +11,17 @@ function setup() { # Create dummy gpg keys export GNUPGHOME=$TESTDIR/skopeo-gpg mkdir --mode=0700 $GNUPGHOME + + # gpg on f30 needs this, otherwise: + # gpg: agent_genkey failed: Inappropriate ioctl for device + # ...but gpg on f29 (and, probably, Ubuntu) doesn't grok this + GPGOPTS='--pinentry-mode loopback' + if gpg --pinentry-mode asdf 2>&1 | grep -qi 'Invalid option'; then + GPGOPTS= + fi + for k in alice bob;do - gpg --batch --pinentry-mode loopback --gen-key --passphrase '' <> $AUTHDIR/htpasswd fi @@ -305,7 +309,7 @@ start_registry() { cp $CERT $TESTDIR/client-auth/ fi - podman run -d --name $name "${reg_args[@]}" registry:2 + $PODMAN run -d --name $name "${reg_args[@]}" registry:2 } # END helpers for starting/stopping registries