From 12f0e2451901a721be2a01d6b1266e3a0e5fc567 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Tue, 14 May 2019 10:30:16 -0600 Subject: [PATCH] systemtest - new set of BATS tests for RHEL8 gating Signed-off-by: Ed Santiago --- systemtest/001-basic.bats | 19 ++ systemtest/010-inspect.bats | 67 +++++ systemtest/020-copy.bats | 79 ++++++ systemtest/030-local-registry-tls.bats | 41 +++ systemtest/040-local-registry-auth.bats | 76 ++++++ systemtest/050-signing.bats | 137 ++++++++++ systemtest/helpers.bash | 318 ++++++++++++++++++++++++ systemtest/run-tests | 19 ++ 8 files changed, 756 insertions(+) create mode 100644 systemtest/001-basic.bats create mode 100644 systemtest/010-inspect.bats create mode 100644 systemtest/020-copy.bats create mode 100644 systemtest/030-local-registry-tls.bats create mode 100644 systemtest/040-local-registry-auth.bats create mode 100644 systemtest/050-signing.bats create mode 100644 systemtest/helpers.bash create mode 100755 systemtest/run-tests diff --git a/systemtest/001-basic.bats b/systemtest/001-basic.bats new file mode 100644 index 00000000..314052fc --- /dev/null +++ b/systemtest/001-basic.bats @@ -0,0 +1,19 @@ +#!/usr/bin/env bats +# +# Simplest set of skopeo tests. If any of these fail, we have serious problems. +# + +load helpers + +# Override standard setup! We don't yet trust anything +function setup() { + : +} + +@test "skopeo version emits reasonable output" { + run_skopeo --version + + expect_output --substring "skopeo version [0-9.]+" +} + +# vim: filetype=sh diff --git a/systemtest/010-inspect.bats b/systemtest/010-inspect.bats new file mode 100644 index 00000000..ee01d123 --- /dev/null +++ b/systemtest/010-inspect.bats @@ -0,0 +1,67 @@ +#!/usr/bin/env bats +# +# Simplest test for skopeo inspect +# + +load helpers + +@test "inspect: basic" { + workdir=$TESTDIR/inspect + + remote_image=docker://quay.io/libpod/alpine_labels:latest + # Inspect remote source, then pull it. There's a small race condition + # in which the remote image can get updated between the inspect and + # the copy; let's just not worry about it. + run_skopeo inspect $remote_image + inspect_remote=$output + + # Now pull it into a directory + run_skopeo copy $remote_image dir:$workdir + expect_output --substring "Getting image source signatures" + expect_output --substring "Writing manifest to image destination" + + # Unpacked contents must include a manifest and version + [ -e $workdir/manifest.json ] + [ -e $workdir/version ] + + # Now run inspect locally + run_skopeo inspect dir:$workdir + inspect_local=$output + + # Each SHA-named file must be listed in the output of 'inspect' + for sha in $(find $workdir -type f | xargs -l1 basename | egrep '^[0-9a-f]{64}$'); do + expect_output --from="$inspect_local" --substring "sha256:$sha" \ + "Locally-extracted SHA file is present in 'inspect'" + done + + # Simple sanity check on 'inspect' output. + # For each of the given keys (LHS of the table below): + # 1) Get local and remote values + # 2) Sanity-check local value using simple expression + # 3) Confirm that local and remote values match. + # + # The reason for (2) is to make sure that we don't compare bad results + # + # The reason for a hardcoded list, instead of 'jq keys', is that RepoTags + # is always empty locally, but a list remotely. + while read key expect; do + local=$(echo "$inspect_local" | jq -r ".$key") + remote=$(echo "$inspect_remote" | jq -r ".$key") + + expect_output --from="$local" --substring "$expect" \ + "local $key is sane" + + expect_output --from="$remote" "$local" \ + "local $key matches remote" + done <$GNUPGHOME/pubkey-$k.gpg + done + + # Registries. The important part here seems to be sigstore, + # because (I guess?) the registry itself has no mechanism + # for storing or validating signatures. + REGISTRIES_D=$TESTDIR/registries.d + mkdir $REGISTRIES_D $TESTDIR/sigstore + cat >$REGISTRIES_D/registries.yaml <$POLICY_JSON </dev/null; then + echo "*** TIMED OUT ***" + false + fi + + if [ -n "$expected_rc" ]; then + if [ "$status" -ne "$expected_rc" ]; then + die "exit code is $status; expected $expected_rc" + fi + fi +} + +######### +# die # Abort with helpful message +######### +function die() { + echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2 + echo "#| FAIL: $*" >&2 + echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2 + false +} + +################### +# expect_output # Compare actual vs expected string; fail if mismatch +################### +# +# Compares $output against the given string argument. Optional second +# argument is descriptive text to show as the error message (default: +# the command most recently run by 'run_skopeo'). This text can be +# useful to isolate a failure when there are multiple identical +# run_skopeo invocations, and the difference is solely in the +# config or setup; see, e.g., run.bats:run-cmd(). +# +# By default we run an exact string comparison; use --substring to +# look for the given string anywhere in $output. +# +# By default we look in "$output", which is set in run_skopeo(). +# To override, use --from="some-other-string" (e.g. "${lines[0]}") +# +# Examples: +# +# expect_output "this is exactly what we expect" +# expect_output "foo=bar" "description of this particular test" +# expect_output --from="${lines[0]}" "expected first line" +# +function expect_output() { + # By default we examine $output, the result of run_skopeo + local actual="$output" + local check_substring= + + # option processing: recognize --from="...", --substring + local opt + for opt; do + local value=$(expr "$opt" : '[^=]*=\(.*\)') + case "$opt" in + --from=*) actual="$value"; shift;; + --substring) check_substring=1; shift;; + --) shift; break;; + -*) die "Invalid option '$opt'" ;; + *) break;; + esac + done + + local expect="$1" + local testname="${2:-${MOST_RECENT_SKOPEO_COMMAND:-[no test name given]}}" + + if [ -z "$expect" ]; then + if [ -z "$actual" ]; then + return + fi + expect='[no output]' + elif [ "$actual" = "$expect" ]; then + return + elif [ -n "$check_substring" ]; then + if [[ "$actual" =~ $expect ]]; then + return + fi + fi + + # This is a multi-line message, which may in turn contain multi-line + # output, so let's format it ourself, readably + local -a actual_split + readarray -t actual_split <<<"$actual" + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| expected: '%s'\n" "$expect" >&2 + printf "#| actual: '%s'\n" "${actual_split[0]}" >&2 + local line + for line in "${actual_split[@]:1}"; do + printf "#| > '%s'\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 + false +} + +####################### +# expect_line_count # Check the expected number of output lines +####################### +# +# ...from the most recent run_skopeo command +# +function expect_line_count() { + local expect="$1" + local testname="${2:-${MOST_RECENT_SKOPEO_COMMAND:-[no test name given]}}" + + local actual="${#lines[@]}" + if [ "$actual" -eq "$expect" ]; then + return + fi + + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| Expected %d lines of output, got %d\n" $expect $actual >&2 + printf "#| Output was:\n" >&2 + local line + for line in "${lines[@]}"; do + printf "#| >%s\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 + false +} + +# END standard helpers for running skopeo and testing results +############################################################################### +# BEGIN helpers for starting/stopping registries + +start_registry() { + local port=5000 + local testuser= + local testpassword= + local create_cert= + + # option processing: recognize --auth + local opt + for opt; do + local value=$(expr "$opt" : '[^=]*=\(.*\)') + case "$opt" in + --port=*) port="$value"; shift;; + --testuser=*) testuser="$value"; shift;; + --testpassword=*) testpassword="$value"; shift;; + --with-cert) create_cert=1; shift;; + -*) die "Invalid option '$opt'" ;; + *) break;; + esac + done + + local name=${1?start_registry() invoked without a NAME} + + # Temp directory must be defined and must exist + [[ -n $TESTDIR && -d $TESTDIR ]] + + AUTHDIR=$TESTDIR/auth + mkdir -p $AUTHDIR + + local -a reg_args=(-v $AUTHDIR:/auth:Z -p $port:5000) + + # Called with --testuser? Create an htpasswd file + if [[ -n $testuser ]]; then + if [[ -z $testpassword ]]; then + die "start_registry() invoked with testuser but no testpassword" + fi + + if ! egrep -q "^$testuser:" $AUTHDIR/htpasswd; then + podman run --rm --entrypoint htpasswd registry:2 \ + -Bbn $testuser $testpassword >> $AUTHDIR/htpasswd + fi + + reg_args+=( + -e REGISTRY_AUTH=htpasswd + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd + -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" + ) + fi + + # Called with --with-cert? Create certificates. + if [[ -n $create_cert ]]; then + CERT=$AUTHDIR/domain.cert + if [ ! -e $CERT ]; then + openssl req -newkey rsa:4096 -nodes -sha256 \ + -keyout $AUTHDIR/domain.key -x509 -days 2 \ + -out $CERT \ + -subj "/C=US/ST=Foo/L=Bar/O=Red Hat, Inc./CN=localhost" + fi + + reg_args+=( + -e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.cert + -e REGISTRY_HTTP_TLS_KEY=/auth/domain.key + ) + fi + + podman run -d --name $name "${reg_args[@]}" registry:2 +} + + +stop_registries() { + if [[ -z $SKOPEO_DEBUG_REGISTRIES ]]; then + podman rm -a -f + + if [[ -n $AUTHDIR ]]; then + rm -rf $AUTHDIR + fi + fi +} + +# END helpers for starting/stopping registries +############################################################################### +# BEGIN miscellaneous tools + +################### +# random_string # Returns a pseudorandom human-readable string +################### +# +# Numeric argument, if present, is desired length of string +# +function random_string() { + local length=${1:-10} + + head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length +} + +# END miscellaneous tools +############################################################################### diff --git a/systemtest/run-tests b/systemtest/run-tests new file mode 100755 index 00000000..004c5f56 --- /dev/null +++ b/systemtest/run-tests @@ -0,0 +1,19 @@ +#!/bin/bash +# +# run-tests - simple wrapper allowing shortcuts on invocation +# + +# FIXME +export SKOPEO_BINARY=${SKOPEO_BINARY:-/usr/bin/skopeo} + +TEST_DIR=$(dirname $0) +TESTS=$TEST_DIR + +for i; do + case "$i" in + *.bats) TESTS=$i ;; + *) TESTS=$(echo $TEST_DIR/*$i*.bats) ;; + esac +done + +bats $TESTS