TMT: add more tests

Signed-off-by: Lokesh Mandvekar <lsm5@redhat.com>
2025-08-31 14:20:21 +00:00 · 2025-07-14 10:48:30 -04:00
parent 3ec5f07881
commit 70334e70c5
10 changed files with 230 additions and 67 deletions
--- a/.packit.yaml
+++ b/.packit.yaml
@@ -21,7 +21,7 @@ files_to_sync:
    dest: plans/
    delete: true
    mkpath: true
-  - src: systemtest/tmt/
+  - src: tmt/
    dest: test/tmt/
    delete: true
    mkpath: true
@@ -47,9 +47,9 @@ jobs:
  - job: copr_build
    trigger: pull_request
    packages: [skopeo-fedora]
-    notifications: &copr_build_failure_notification
+    notifications: &packit_failure_notification
      failure_comment:
-        message: "Ephemeral COPR build failed. @containers/packit-build please check."
+        message: "Packit jobs failed. @containers/packit-build please check."
    targets: &fedora_copr_targets
      - fedora-all-x86_64
      - fedora-all-aarch64
@@ -62,7 +62,7 @@ jobs:
  - job: copr_build
    trigger: ignore
    packages: [skopeo-eln]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
    targets:
      fedora-eln-x86_64:
        additional_repos:
@@ -76,7 +76,7 @@ jobs:
  - job: copr_build
    trigger: ignore
    packages: [skopeo-centos]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
    targets: &centos_copr_targets
      - centos-stream-9-x86_64
      - centos-stream-9-aarch64
@@ -88,34 +88,83 @@ jobs:
  - job: copr_build
    trigger: commit
    packages: [skopeo-fedora]
-    notifications:
-      failure_comment:
-        message: "podman-next COPR build failed. @containers/packit-build please check."
    branch: main
    owner: rhcontainerbot
    project: podman-next
    enable_net: true

-  # Tests on Fedora for main branch
+  # System tests on Fedora
  - job: tests
    trigger: pull_request
    packages: [skopeo-fedora]
-    notifications: &test_failure_notification
-      failure_comment:
-        message: "Tests failed. @containers/packit-build please check."
+    notifications: *packit_failure_notification
    targets: *fedora_copr_targets
-    tf_extra_params:
+    tf_extra_params: &extra_dnf_repos
      environments:
        - artifacts:
          - type: repository-file
            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+    tmt_plan: "/plans/rpm/system"
+    identifier: "system"
+
+  # ostree-rs-ext tests on Fedora
+  - job: tests
+    trigger: pull_request
+    packages: [skopeo-fedora]
+    notifications: *packit_failure_notification
+    targets: *fedora_copr_targets
+    tf_extra_params: *extra_dnf_repos
+    tmt_plan: "/plans/rpm/ostree-rs-ext"
+    identifier: "ostree-rs-ext"
+
+  # Integration tests on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    packages: [skopeo-fedora]
+    notifications: *packit_failure_notification
+    targets:
+      # x86_6 only because of quay.io/coreos/11bot
+      # See: integration/proxy_test.go
+      - fedora-all
+    tf_extra_params: *extra_dnf_repos
+    tmt_plan: "/plans/no-rpm/integration"
+    identifier: "integration"
+
+  # Unit tests on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: &fedora_no_rpm_targets
+      - fedora-latest-stable
+    tmt_plan: "/plans/no-rpm/unit"
+    identifier: "unit"
+
+  # Validate test on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: *fedora_no_rpm_targets
+    tmt_plan: "/plans/no-rpm/validate"
+    identifier: "validate"
+
+  # Test w/ opengpg static linked on Fedora
+  - job: tests
+    trigger: pull_request
+    skip_build: true
+    notifications: *packit_failure_notification
+    targets: *fedora_no_rpm_targets
+    tmt_plan: "/plans/no-rpm/opengpg"
+    identifier: "opengpg"

  # Tests on CentOS Stream for main branch
  # Ignore until golang is updated in distro buildroot to go 1.23.3+
  - job: tests
    trigger: ignore
    packages: [skopeo-centos]
-    notifications: *test_failure_notification
+    notifications: *packit_failure_notification
    targets: *centos_copr_targets
    tf_extra_params:
      environments:
--- a/6
+++ b/6
@@ -201,7 +201,13 @@ test-integration:
 	$(CONTAINER_CMD) --security-opt label=disable --cap-add=cap_mknod -v $(CURDIR):$(CONTAINER_GOSRC) -w $(CONTAINER_GOSRC) $(SKOPEO_CIDEV_CONTAINER_FQIN) \
 		$(MAKE) test-integration-local

+<<<<<<< HEAD
 # Expects binary in $PATH, so better to depend on install-binary target
+=======
+
+# Intended for CI, assumed to be running in quay.io/libpod/skopeo_cidev container.
+# Expects binary in PATH, so better to depend on install-binary target
+>>>>>>> 5bf4e23d (TMT: add more tests)
 test-integration-local: install-binary
 	hack/test-integration.sh

--- a/hack/tmt/bats-setup.sh
+++ b/hack/tmt/bats-setup.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+# Install bats
+# https://bats-core.readthedocs.io/en/stable/installation.html
+
+BATS_TMPDIR=$(mktemp -d)
+pushd "$BATS_TMPDIR"
+
+BATS_VERSION=1.12.0
+curl -ssfL https://github.com/bats-core/bats-core/archive/refs/tags/v"$BATS_VERSION".tar.gz | tar -xz
+pushd bats-core-"$BATS_VERSION"
+./install.sh /usr
+popd
+popd
--- a/hack/tmt/dnf-repo-setup.sh
+++ b/hack/tmt/dnf-repo-setup.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+COPR_REPO_FILE=$(compgen -G "/etc/yum.repos.d/*podman-next*.repo")
+if [[ -n "$COPR_REPO_FILE" ]]; then
+    # shellcheck disable=SC2016
+    sed -i -n '/^priority=/!p;$apriority=1' "${COPR_REPO_FILE}"
+fi
+dnf -y upgrade --allowerasing
--- a/plans/main.fmf
+++ b/plans/main.fmf
@@ -1,26 +0,0 @@
-discover:
-    how: fmf
-execute:
-    how: tmt
-prepare:
-    - when: distro == centos-stream or distro == rhel
-      how: shell
-      script: |
-        # Install bats
-        # https://bats-core.readthedocs.io/en/stable/installation.html
-        BATS_VERSION=1.12.0
-        curl -L https://github.com/bats-core/bats-core/archive/refs/tags/v"$BATS_VERSION".tar.gz | tar -xz
-        pushd bats-core-"$BATS_VERSION"
-        ./install.sh /usr
-        popd
-        rm -rf bats-core-"$BATS_VERSION"
-      order: 10
-    - when: initiator == packit
-      how: shell
-      script: |
-        COPR_REPO_FILE="/etc/yum.repos.d/*podman-next*.repo"
-        if compgen -G $COPR_REPO_FILE > /dev/null; then
-            sed -i -n '/^priority=/!p;$apriority=1' $COPR_REPO_FILE
-        fi
-        dnf -y upgrade --allowerasing
-      order: 20
--- a/plans/no-rpm.fmf
+++ b/plans/no-rpm.fmf
@@ -0,0 +1,25 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+
+/integration:
+    discover+:
+        test: /tmt/integration
+
+/validate:
+    discover+:
+        test: /tmt/validate
+
+/unit:
+    discover+:
+        test: /tmt/unit
+
+/opengpg:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/opengpg
--- a/plans/rpm.fmf
+++ b/plans/rpm.fmf
@@ -0,0 +1,22 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+prepare:
+    - when: initiator == packit
+      how: shell
+      script: bash hack/tmt/dnf-repo-setup.sh
+      order: 5
+
+/ostree-rs-ext:
+    discover+:
+        test: /tmt/ostree-rs-ext
+
+/system:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/system
--- a/systemtest/tmt/main.fmf
+++ b/systemtest/tmt/main.fmf
@@ -1,14 +0,0 @@
-require:
-    - skopeo-tests
-
-environment:
-    SKOPEO_BINARY: /usr/bin/skopeo
-
-adjust:
-    - when: initiator != "packit"
-      environment+:
-        RELEASE_TESTING: true
-
-summary: System test
-test: bash ./test.sh
-duration: 60m
--- a/systemtest/tmt/test.sh
+++ b/systemtest/tmt/test.sh
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-set -exo pipefail
-
-uname -r
-
-rpm -q \
-    bats \
-    containers-common \
-    skopeo \
-    skopeo-tests \
-
-bats /usr/share/skopeo/test/system
--- a/tmt/main.fmf
+++ b/tmt/main.fmf
@@ -0,0 +1,88 @@
+require:
+    # Some of these are only required for non-rpm tests but it's much simpler
+    # to list them as common deps. Shouldn't be too much of a burden for rpm
+    # jobs.
+    - btrfs-progs-devel
+    - containers-common
+    - docker-distribution
+    - go-md2man
+    - golang
+    - gpgme-devel
+    - httpd-tools
+    - jq
+    - make
+
+/integration:
+    summary: Integration test
+    test: |
+        rpm -q containers-common
+        make -C $TMT_TREE test-integration-local
+    duration: 20m
+
+/ostree-rs-ext:
+    /build:
+        summary: ostree-rs-ext build
+        test: bash $TMT_TREE/hack/test-ostree.sh build
+
+    /test:
+        summary: ostree-rs-ext test
+        test: bash $TMT_TREE/hack/test-ostree.sh test
+        require+:
+            - ostree
+
+/opengpg:
+    # w/ opengpg will only be tested upstream, so we don't care about setting
+    # $RELEASE_TESTING.
+
+    enabled: false
+    adjust:
+        enabled: true
+        when: initiator == packit
+
+    /validate+unit:
+        summary: System test with opengpg
+        test: |
+            make -C $TMT_TREE BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $TMT_TREE test-all-local
+
+    /integration:
+        summary: Integration test with opengpg
+        test: |
+            make -C $TMT_TREE BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $TMT_TREE test-integration-local
+        duration: 30m
+
+    /system:
+        summary: System tests with opengpg
+        test: |
+            make -C $TMT_TREE BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $TMT_TREE test-system-local
+        duration: 30m
+        require+:
+            - bats
+
+/system:
+    summary: System test
+    test: |
+        rpm -q containers-common skopeo
+        make -C $TMT_TREE test-system-local
+    environment:
+        SKOPEO_BINARY: /usr/bin/skopeo
+    adjust:
+        - when: initiator != "packit"
+          environment+:
+            RELEASE_TESTING: true
+    duration: 60m
+
+/unit:
+    summary: Unit test
+    test: |
+        make -C $TMT_TREE
+        make -C $TMT_TREE test-unit-local
+
+/validate:
+    summary: Validate test
+    test: |
+        make -C $TMT_TREE
+        make -C $TMT_TREE validate-docs validate-local
+