diff --git a/Dockerfile b/Dockerfile index 3cf2be4f..28d82980 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-go-md gnupg \ # OpenShift deps which tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \ + bats jq podman \ && dnf clean all # Install two versions of the registry. The first is an older version that diff --git a/Makefile b/Makefile index 8f431701..6e5df788 100644 --- a/Makefile +++ b/Makefile @@ -138,12 +138,23 @@ install-completions: shell: build-container $(CONTAINER_RUN) bash -check: validate test-unit test-integration +check: validate test-unit test-integration test-system # The tests can run out of entropy and block in containers, so replace /dev/random. test-integration: build-container $(CONTAINER_RUN) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-integration' +# complicated set of options needed to run podman-in-podman +test-system: build-container + DTEMP=$(shell mktemp -d --tmpdir=/var/tmp podman-tmp.XXXXXX); \ + $(CONTAINER_CMD) --privileged --net=host \ + -v $$DTEMP:/var/lib/containers:Z \ + "$(IMAGE)" \ + bash -c 'BUILDTAGS="$(BUILDTAGS)" hack/make.sh test-system'; \ + rc=$$?; \ + $(RM) -rf $$DTEMP; \ + exit $$rc + test-unit: build-container # Just call (make test unit-local) here instead of worrying about environment differences, e.g. GO15VENDOREXPERIMENT. $(CONTAINER_RUN) make test-unit-local BUILDTAGS='$(BUILDTAGS)' diff --git a/hack/make/test-system b/hack/make/test-system new file mode 100755 index 00000000..bc881e17 --- /dev/null +++ b/hack/make/test-system @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Before running podman for the first time, make sure +# to set storage to vfs (not overlay): podman-in-podman +# doesn't work with overlay. And, disable mountopt, +# which causes error with vfs. +sed -i \ + -e 's/^driver\s*=.*/driver = "vfs"/' \ + -e 's/^mountopt/#mountopt/' \ + /etc/containers/storage.conf + +# Build skopeo, install into /usr/bin +make binary-local ${BUILDTAGS:+BUILDTAGS="$BUILDTAGS"} +make install + +# Run tests +SKOPEO_BINARY=/usr/bin/skopeo bats --tap systemtest diff --git a/systemtest/001-basic.bats b/systemtest/001-basic.bats new file mode 100644 index 00000000..314052fc --- /dev/null +++ b/systemtest/001-basic.bats @@ -0,0 +1,19 @@ +#!/usr/bin/env bats +# +# Simplest set of skopeo tests. If any of these fail, we have serious problems. +# + +load helpers + +# Override standard setup! We don't yet trust anything +function setup() { + : +} + +@test "skopeo version emits reasonable output" { + run_skopeo --version + + expect_output --substring "skopeo version [0-9.]+" +} + +# vim: filetype=sh diff --git a/systemtest/010-inspect.bats b/systemtest/010-inspect.bats new file mode 100644 index 00000000..ee01d123 --- /dev/null +++ b/systemtest/010-inspect.bats @@ -0,0 +1,67 @@ +#!/usr/bin/env bats +# +# Simplest test for skopeo inspect +# + +load helpers + +@test "inspect: basic" { + workdir=$TESTDIR/inspect + + remote_image=docker://quay.io/libpod/alpine_labels:latest + # Inspect remote source, then pull it. There's a small race condition + # in which the remote image can get updated between the inspect and + # the copy; let's just not worry about it. + run_skopeo inspect $remote_image + inspect_remote=$output + + # Now pull it into a directory + run_skopeo copy $remote_image dir:$workdir + expect_output --substring "Getting image source signatures" + expect_output --substring "Writing manifest to image destination" + + # Unpacked contents must include a manifest and version + [ -e $workdir/manifest.json ] + [ -e $workdir/version ] + + # Now run inspect locally + run_skopeo inspect dir:$workdir + inspect_local=$output + + # Each SHA-named file must be listed in the output of 'inspect' + for sha in $(find $workdir -type f | xargs -l1 basename | egrep '^[0-9a-f]{64}$'); do + expect_output --from="$inspect_local" --substring "sha256:$sha" \ + "Locally-extracted SHA file is present in 'inspect'" + done + + # Simple sanity check on 'inspect' output. + # For each of the given keys (LHS of the table below): + # 1) Get local and remote values + # 2) Sanity-check local value using simple expression + # 3) Confirm that local and remote values match. + # + # The reason for (2) is to make sure that we don't compare bad results + # + # The reason for a hardcoded list, instead of 'jq keys', is that RepoTags + # is always empty locally, but a list remotely. + while read key expect; do + local=$(echo "$inspect_local" | jq -r ".$key") + remote=$(echo "$inspect_remote" | jq -r ".$key") + + expect_output --from="$local" --substring "$expect" \ + "local $key is sane" + + expect_output --from="$remote" "$local" \ + "local $key matches remote" + done <&1 | grep -qi 'Invalid option'; then + GPGOPTS= + fi + + for k in alice bob;do + gpg --batch $GPGOPTS --gen-key --passphrase '' <$GNUPGHOME/pubkey-$k.gpg + done + + # Registries. The important part here seems to be sigstore, + # because (I guess?) the registry itself has no mechanism + # for storing or validating signatures. + REGISTRIES_D=$TESTDIR/registries.d + mkdir $REGISTRIES_D $TESTDIR/sigstore + cat >$REGISTRIES_D/registries.yaml <$POLICY_JSON </dev/null; then + echo "*** TIMED OUT ***" + false + fi + + if [ -n "$expected_rc" ]; then + if [ "$status" -ne "$expected_rc" ]; then + die "exit code is $status; expected $expected_rc" + fi + fi +} + +######### +# die # Abort with helpful message +######### +function die() { + echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2 + echo "#| FAIL: $*" >&2 + echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2 + false +} + +################### +# expect_output # Compare actual vs expected string; fail if mismatch +################### +# +# Compares $output against the given string argument. Optional second +# argument is descriptive text to show as the error message (default: +# the command most recently run by 'run_skopeo'). This text can be +# useful to isolate a failure when there are multiple identical +# run_skopeo invocations, and the difference is solely in the +# config or setup; see, e.g., run.bats:run-cmd(). +# +# By default we run an exact string comparison; use --substring to +# look for the given string anywhere in $output. +# +# By default we look in "$output", which is set in run_skopeo(). +# To override, use --from="some-other-string" (e.g. "${lines[0]}") +# +# Examples: +# +# expect_output "this is exactly what we expect" +# expect_output "foo=bar" "description of this particular test" +# expect_output --from="${lines[0]}" "expected first line" +# +function expect_output() { + # By default we examine $output, the result of run_skopeo + local actual="$output" + local check_substring= + + # option processing: recognize --from="...", --substring + local opt + for opt; do + local value=$(expr "$opt" : '[^=]*=\(.*\)') + case "$opt" in + --from=*) actual="$value"; shift;; + --substring) check_substring=1; shift;; + --) shift; break;; + -*) die "Invalid option '$opt'" ;; + *) break;; + esac + done + + local expect="$1" + local testname="${2:-${MOST_RECENT_SKOPEO_COMMAND:-[no test name given]}}" + + if [ -z "$expect" ]; then + if [ -z "$actual" ]; then + return + fi + expect='[no output]' + elif [ "$actual" = "$expect" ]; then + return + elif [ -n "$check_substring" ]; then + if [[ "$actual" =~ $expect ]]; then + return + fi + fi + + # This is a multi-line message, which may in turn contain multi-line + # output, so let's format it ourself, readably + local -a actual_split + readarray -t actual_split <<<"$actual" + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| expected: '%s'\n" "$expect" >&2 + printf "#| actual: '%s'\n" "${actual_split[0]}" >&2 + local line + for line in "${actual_split[@]:1}"; do + printf "#| > '%s'\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 + false +} + +####################### +# expect_line_count # Check the expected number of output lines +####################### +# +# ...from the most recent run_skopeo command +# +function expect_line_count() { + local expect="$1" + local testname="${2:-${MOST_RECENT_SKOPEO_COMMAND:-[no test name given]}}" + + local actual="${#lines[@]}" + if [ "$actual" -eq "$expect" ]; then + return + fi + + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| Expected %d lines of output, got %d\n" $expect $actual >&2 + printf "#| Output was:\n" >&2 + local line + for line in "${lines[@]}"; do + printf "#| >%s\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 + false +} + +# END standard helpers for running skopeo and testing results +############################################################################### +# BEGIN helpers for starting/stopping registries + +#################### +# start_registry # Run a local registry container +#################### +# +# Usage: start_registry [OPTIONS] NAME +# +# OPTIONS +# --port=NNNN Port to listen on (default: 5000) +# --testuser=XXX Require authentication; this is the username +# --testpassword=XXX ...and the password (these two go together) +# --with-cert Create a cert for running with TLS (not working) +# +# NAME is the container name to assign. +# +start_registry() { + local port=5000 + local testuser= + local testpassword= + local create_cert= + + # option processing: recognize options for running the registry + # in different modes. + local opt + for opt; do + local value=$(expr "$opt" : '[^=]*=\(.*\)') + case "$opt" in + --port=*) port="$value"; shift;; + --testuser=*) testuser="$value"; shift;; + --testpassword=*) testpassword="$value"; shift;; + --with-cert) create_cert=1; shift;; + -*) die "Invalid option '$opt'" ;; + *) break;; + esac + done + + local name=${1?start_registry() invoked without a NAME} + + # Temp directory must be defined and must exist + [[ -n $TESTDIR && -d $TESTDIR ]] + + AUTHDIR=$TESTDIR/auth + mkdir -p $AUTHDIR + + local -a reg_args=(-v $AUTHDIR:/auth:Z -p $port:5000) + + # cgroup option necessary under podman-in-podman (CI tests), + # and doesn't seem to do any harm otherwise. + PODMAN="podman --cgroup-manager=cgroupfs" + + # Called with --testuser? Create an htpasswd file + if [[ -n $testuser ]]; then + if [[ -z $testpassword ]]; then + die "start_registry() invoked with testuser but no testpassword" + fi + + if ! egrep -q "^$testuser:" $AUTHDIR/htpasswd; then + $PODMAN run --rm --entrypoint htpasswd registry:2 \ + -Bbn $testuser $testpassword >> $AUTHDIR/htpasswd + fi + + reg_args+=( + -e REGISTRY_AUTH=htpasswd + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd + -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" + ) + fi + + # Called with --with-cert? Create certificates. + if [[ -n $create_cert ]]; then + CERT=$AUTHDIR/domain.crt + if [ ! -e $CERT ]; then + openssl req -newkey rsa:4096 -nodes -sha256 \ + -keyout $AUTHDIR/domain.key -x509 -days 2 \ + -out $CERT \ + -subj "/C=US/ST=Foo/L=Bar/O=Red Hat, Inc./CN=localhost" + fi + + reg_args+=( + -e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.crt + -e REGISTRY_HTTP_TLS_KEY=/auth/domain.key + ) + + # Copy .crt file to a directory *without* the .key one, so we can + # test the client. (If client sees a matching .key file, it fails) + # Thanks to Miloslav Trmac for this hint. + mkdir -p $TESTDIR/client-auth + cp $CERT $TESTDIR/client-auth/ + fi + + $PODMAN run -d --name $name "${reg_args[@]}" registry:2 +} + +# END helpers for starting/stopping registries +############################################################################### +# BEGIN miscellaneous tools + +################### +# random_string # Returns a pseudorandom human-readable string +################### +# +# Numeric argument, if present, is desired length of string +# +function random_string() { + local length=${1:-10} + + head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length +} + +# END miscellaneous tools +############################################################################### diff --git a/systemtest/run-tests b/systemtest/run-tests new file mode 100755 index 00000000..21763494 --- /dev/null +++ b/systemtest/run-tests @@ -0,0 +1,16 @@ +#!/bin/bash +# +# run-tests - simple wrapper allowing shortcuts on invocation +# + +TEST_DIR=$(dirname $0) +TESTS=$TEST_DIR + +for i; do + case "$i" in + *.bats) TESTS=$i ;; + *) TESTS=$(echo $TEST_DIR/*$i*.bats) ;; + esac +done + +bats $TESTS