TMT: add more tests

Signed-off-by: Lokesh Mandvekar <lsm5@redhat.com>

Author: lsm5

.packit.yaml

@@ -32,9 +32,9 @@ jobs:
   - job: copr_build
     trigger: pull_request
     packages: [skopeo-fedora]
-    notifications: &copr_build_failure_notification
+    notifications: &packit_failure_notification
       failure_comment:
-        message: "Ephemeral COPR build failed. @containers/packit-build please check."
+        message: "Packit jobs failed. @containers/packit-build please check."
     targets: &fedora_copr_targets
       - fedora-all-x86_64
       - fedora-all-aarch64
@@ -46,8 +46,9 @@ jobs:
   - job: copr_build
     trigger: pull_request
     packages: [skopeo-eln]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
     targets: &eln_copr_targets
+    targets:
       fedora-eln-x86_64:
         additional_repos:
           - "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/x86_64/"
@@ -59,7 +60,7 @@ jobs:
   - job: copr_build
     trigger: pull_request
     packages: [skopeo-centos]
-    notifications: *copr_build_failure_notification
+    notifications: *packit_failure_notification
     targets: &centos_copr_targets
       - centos-stream-9-x86_64
       - centos-stream-9-aarch64
@@ -71,51 +72,36 @@ jobs:
   - job: copr_build
     trigger: commit
     packages: [skopeo-fedora]
-    notifications:
-      failure_comment:
-        message: "podman-next COPR build failed. @containers/packit-build please check."
     branch: main
     owner: rhcontainerbot
     project: podman-next
     enable_net: true
 
-  # Tests on Fedora for main branch
+  # System tests on Fedora
   - job: tests
     trigger: pull_request
     packages: [skopeo-fedora]
-    notifications: &test_failure_notification
-      failure_comment:
-        message: "Tests failed. @containers/packit-build please check."
+    notifications: *packit_failure_notification
     targets: *fedora_copr_targets
-    tf_extra_params:
+    tf_extra_params: &extra_dnf_repos
       environments:
         - artifacts:
           - type: repository-file
             id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+    tmt_plan: "/plans/rpm/system"
+    identifier: "system"
 
-  # Tests on ELN for main branch
-  - job: tests
-    trigger: pull_request
-    packages: [skopeo-eln]
-    notifications: *test_failure_notification
-    targets: *eln_copr_targets
-    tf_extra_params:
-      environments:
-        - artifacts:
-          - type: repository-file
-            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-eln/rhcontainerbot-podman-next-fedora-eln.repo
-
-  # Tests on CentOS Stream for main branch
+  # Integration tests on Fedora
   - job: tests
     trigger: pull_request
-    packages: [skopeo-centos]
-    notifications: *test_failure_notification
-    targets: *centos_copr_targets
-    tf_extra_params:
-      environments:
-        - artifacts:
-          - type: repository-file
-            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/centos-stream-$releasever/rhcontainerbot-podman-next-centos-stream-$releasever.repo
+    skip_build: true
+    packages: [skopeo-fedora]
+    notifications: *packit_failure_notification
+    targets:
+      - fedora-rawhide
+    tf_extra_params: *extra_dnf_repos
+    tmt_plan: "/plans/no-rpm/integration"
+    identifier: "integration"
 
   # Sync to Fedora
   - job: propose_downstream

Makefile

@@ -200,7 +200,6 @@ test-integration:
 	$(CONTAINER_CMD) --security-opt label=disable --cap-add=cap_mknod -v $(CURDIR):$(CONTAINER_GOSRC) -w $(CONTAINER_GOSRC) $(SKOPEO_CIDEV_CONTAINER_FQIN) \
 		$(MAKE) test-integration-local
 
-
 # Helper target to set up SKOPEO_BINARY variable for local test targets
 .eval-skopeo-binary: $(if $(SKOPEO_BINARY),,bin/skopeo)
 	$(eval SKOPEO_BINARY := $(or $(SKOPEO_BINARY),./bin/skopeo))

hack/tmt/bats-setup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+# Install bats
+# https://bats-core.readthedocs.io/en/stable/installation.html
+
+BATS_TMPDIR=$(mktemp -d)
+pushd "$BATS_TMPDIR"
+
+BATS_VERSION=1.12.0
+curl -ssfL https://github.com/bats-core/bats-core/archive/refs/tags/v"$BATS_VERSION".tar.gz | tar -xz
+pushd bats-core-"$BATS_VERSION"
+./install.sh /usr
+popd
+popd

hack/tmt/dnf-repo-setup.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+COPR_REPO_FILE=$(compgen -G "/etc/yum.repos.d/*podman-next*.repo")
+if [[ -n "$COPR_REPO_FILE" ]]; then
+    # shellcheck disable=SC2016
+    sed -i -n '/^priority=/!p;$apriority=1' "${COPR_REPO_FILE}"
+fi
+dnf -y upgrade --allowerasing

plans/main.fmf

@@ -0,0 +1,25 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+
+/integration:
+    discover+:
+        test: /tmt/integration
+
+/validate:
+    discover+:
+        test: /tmt/validate
+
+/unit:
+    discover+:
+        test: /tmt/unit
+
+/opengpg:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/opengpg

plans/no-rpm.fmf

@@ -0,0 +1,33 @@
+discover:
+    how: fmf
+
+execute:
+    how: tmt
+
+prepare:
+    - when: initiator == packit
+      how: shell
+      script: bash ./hack/tmt/dnf-repo-setup.sh
+      order: 20
+    - name: Disable installing everything from srpm
+      how: install
+      exclude:
+        - ".*"
+    - name: Install the main package
+      how: install
+      package:
+        # skopeo-tests installs skopeo and other deps
+        - skopeo-tests
+
+/ostree-rs-ext:
+    discover+:
+        test: /tmt/ostree-rs-ext
+
+/system:
+    prepare+:
+        - when: distro == centos-stream or distro == rhel
+          how: shell
+          script: bash hack/tmt/bats-setup.sh
+          order: 10
+    discover+:
+        test: /tmt/system

plans/rpm.fmf

@@ -0,0 +1,88 @@
+require:
+    # Some of these are only required for non-rpm tests but it's much simpler
+    # to list them as common deps. Shouldn't be too much of a burden for rpm
+    # jobs.
+    - btrfs-progs-devel
+    - docker-distribution
+    - go-md2man
+    - golang
+    - gpgme-devel
+    - git-core
+    - make
+    - skopeo-tests
+
+/integration:
+    summary: Integration test
+    test: |
+        rpm -q containers-common
+        make -C $(git rev-parse --show-toplevel) test-integration-local
+    duration: 20m
+
+/ostree-rs-ext:
+    /build:
+        summary: ostree-rs-ext build
+        test: bash $(git rev-parse --show-toplevel)/hack/test-ostree.sh build
+
+    /test:
+        summary: ostree-rs-ext test
+        test: |
+            rpm -q ostree
+            bash $(git rev-parse --show-toplevel)/hack/test-ostree.sh test
+        require+:
+            - ostree
+
+/opengpg:
+    # w/ opengpg will only be tested upstream, so we don't care about setting
+    # $RELEASE_TESTING.
+
+    enabled: false
+    adjust:
+        enabled: true
+        when: initiator == packit
+
+    /validate+unit:
+        summary: System test with opengpg
+        test: |
+            make -C $(git rev-parse --show-toplevel) BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $(git rev-parse --show-toplevel) test-all-local
+
+    /integration:
+        summary: Integration test with opengpg
+        test: |
+            make -C $(git rev-parse --show-toplevel) BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $(git rev-parse --show-toplevel) test-integration-local
+        duration: 30m
+
+    /system:
+        summary: System tests with opengpg
+        test: |
+            make -C $(git rev-parse --show-toplevel) BUILDTAGS+=" containers_image_openpgp" bin/skopeo
+            make -C $(git rev-parse --show-toplevel) test-system-local
+        duration: 30m
+        require+:
+            - bats
+
+/system:
+    summary: System test
+    test: |
+        rpm -q containers-common skopeo
+        make -C $(git rev-parse --show-toplevel) test-system-local
+    environment:
+        SKOPEO_BINARY: /usr/bin/skopeo
+    adjust:
+        - when: initiator != "packit"
+          environment+:
+            RELEASE_TESTING: true
+    duration: 60m
+
+/unit:
+    summary: Unit test
+    test: |
+        make -C $(git rev-parse --show-toplevel)
+        make -C $(git rev-parse --show-toplevel) test-unit-local
+
+/validate:
+    summary: Validate test
+    test: |
+        make -C $(git rev-parse --show-toplevel)
+        make -C $(git rev-parse --show-toplevel) validate-docs validate-local