pants ci: add integration tests jobs (#6273)

cognifloyd · web-flow · commit e74d7c9f869d · 2024-12-05T00:20:26.000-06:00
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -22,6 +22,19 @@ on:
   #  # run every night at midnight
   #  - cron:  '0 0 * * *'
 
+env:
+  COLUMNS: '120'
+
+  # Tell StackStorm that we are indeed in CI mode, using our CI-provider agnostic var.
+  ST2_CI: 'true'
+
+  # GitHub Actions uses the 'runner' user, so use that instead of stanley.
+  ST2TESTS_SYSTEM_USER: 'runner'
+
+  # This is the host:port provided by services.redis
+  ST2TESTS_REDIS_HOST: '127.0.0.1'
+  ST2TESTS_REDIS_PORT: '6379'
+
 jobs:
   pants-plugins-tests:
     name: 'Pants Plugins Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
@@ -63,9 +76,6 @@ jobs:
          ports:
            - 6379:6379/tcp
 
-    env:
-      COLUMNS: '120'
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -99,11 +109,6 @@ jobs:
           gha-cache-key: cache0-py${{ matrix.python.version }}
 
       - name: Test pants-plugins
-        env:
-          # Github Actions uses the 'runner' user, so use that instead of stanley.
-          ST2TESTS_SYSTEM_USER: 'runner'
-          ST2TESTS_REDIS_HOST: '127.0.0.1'
-          ST2TESTS_REDIS_PORT: '6379'
         run: |
           pants test pants-plugins/::
 
@@ -161,9 +166,6 @@ jobs:
          ports:
            - 6379:6379/tcp
 
-    env:
-      COLUMNS: '120'
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -197,11 +199,6 @@ jobs:
           gha-cache-key: cache0-py${{ matrix.python.version }}
 
       - name: Unit Tests
-        env:
-          # Github Actions uses the 'runner' user, so use that instead of stanley.
-          ST2TESTS_SYSTEM_USER: 'runner'
-          ST2TESTS_REDIS_HOST: '127.0.0.1'
-          ST2TESTS_REDIS_PORT: '6379'
         run: >
           pants
           --python-bootstrap-search-path=[]
@@ -257,9 +254,6 @@ jobs:
          ports:
            - 6379:6379/tcp
 
-    env:
-      COLUMNS: '120'
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -293,11 +287,6 @@ jobs:
           gha-cache-key: cache0-py${{ matrix.python.version }}
 
       - name: Pack Tests
-        env:
-          # Github Actions uses the 'runner' user, so use that instead of stanley.
-          ST2TESTS_SYSTEM_USER: 'runner'
-          ST2TESTS_REDIS_HOST: '127.0.0.1'
-          ST2TESTS_REDIS_PORT: '6379'
         run: >
           pants
           --python-bootstrap-search-path=[]
@@ -312,7 +301,243 @@ jobs:
           path: .pants.d/pants.log
         if: always()  # We want the log even on failures.
 
-  #integration-tests: TODO: run integration tests
+  integration-tests:
+    name: 'Integration Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # NOTE: We need to use full Python version as part of Python deps cache key otherwise
+        # setup virtualenv step will fail.
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+
+    services:
+      mongo:
+        image: mongo:7.0
+        ports:
+          - 27017:27017
+
+      # In GHA, these services are started first before the code is checked out.
+      # We use bitnami images to facilitate reconfiguring RabbitMQ during integration tests.
+      # We rely on custom config and SSL certs that are in the repo.
+      # Many images require config in env vars (which we can't change during the test job)
+      # or they require config in entrypoint args (which we can't override for GHA services)
+      # bitnami builds ways to get config files from mounted volumes.
+      rabbitmq:
+        image: bitnami/rabbitmq:3.8
+        volumes:
+          - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
+        env:
+          # tell bitnami/rabbitmq to enable this by default
+          RABBITMQ_PLUGINS: rabbitmq_management
+          RABBITMQ_USERNAME: guest
+          RABBITMQ_PASSWORD: guest
+          RABBITMQ_LOGS: '-'
+          # bitnami's default relative limit was too high
+          RABBITMQ_DISK_FREE_ABSOLUTE_LIMIT: 50MB
+
+        # These are strictly docker options, not entrypoint args (GHA restriction)
+        options: >-
+          --name rabbitmq
+        ports:
+          # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
+          # host_port:container_port/protocol
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15672:15672/tcp # Management: HTTP, CLI
+          #- 15671:15671/tcp # Management: SSL port
+          #- 25672:25672/tcp # inter-node or CLI
+          #- 4369:4369/tcp   # epmd
+
+      redis:
+         # Docker Hub image
+         image: redis
+         # Set health checks to wait until redis has started
+         options: >-
+           --name "redis"
+           --health-cmd "redis-cli ping"
+           --health-interval 10s
+           --health-timeout 5s
+           --health-retries 5
+         ports:
+           - 6379:6379/tcp
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # a test uses a submodule, and pants needs access to it to calculate deps.
+          submodules: 'recursive'
+          # sadly, the submodule will only have fetch-depth=1, which is what we want
+          # for st2.git, but not for the submodules. We still want actions/checkout
+          # to do the initial checkout, however, so that it adds auth for fetching
+          # in the submodule.
+
+      - name: Fetch repository submodules
+        run: |
+          git submodule status
+          git submodule foreach 'git fetch --all --tags'
+          git submodule foreach 'git tag'
+
+      - name: 'Set up Python (${{ matrix.python.version }})'
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '${{ matrix.python.version }}'
+
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
+
+      - name: Initialize Pants and its GHA caches
+        uses: ./.github/actions/init-pants
+        with:
+          # To ignore a bad cache, bump the cache* integer.
+          gha-cache-key: cache0-py${{ matrix.python.version }}
+
+      # This is only required for st2common/tests/integration/test_rabbitmq_ssl_listener.py
+      - name: Reconfigure RabbitMQ
+        # bitnami image allows (see bitnami/rabbitmq readme):
+        # Here we're copying a rabbitmq.config file which won't do anything.
+        # We need to switch to custom.conf or advanced.config.
+        timeout-minutes: 2  # may die if rabbitmq fails to start
+        run: |
+          ./scripts/github/configure-rabbitmq.sh
+
+      - name: Integration Tests
+        run: >
+          pants
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          --tag=integration
+          --tag=-st2cluster
+          test '::'
+
+      - name: Upload pants log
+        uses: actions/upload-artifact@v4
+        with:
+          name: pants-log-py${{ matrix.python.version }}-integration-tests
+          path: .pants.d/pants.log
+        if: always()  # We want the log even on failures.
+
+  integration-st2cluster-tests:
+    name: 'Integration Tests (Full Cluster)- Python ${{ matrix.python.version-short }}'
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # NOTE: We need to use full Python version as part of Python deps cache key otherwise
+        # setup virtualenv step will fail.
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+
+    services:
+      mongo:
+        image: mongo:7.0
+        ports:
+          - 27017:27017
+
+      rabbitmq:
+        image: rabbitmq:3.8-management
+        options: >-
+          --name rabbitmq
+        ports:
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15672:15672/tcp # Management: HTTP, CLI
+
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --name "redis"
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379/tcp
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # a test uses a submodule, and pants needs access to it to calculate deps.
+          submodules: 'recursive'
+          # sadly, the submodule will only have fetch-depth=1, which is what we want
+          # for st2.git, but not for the submodules. We still want actions/checkout
+          # to do the initial checkout, however, so that it adds auth for fetching
+          # in the submodule.
+
+      - name: Fetch repository submodules
+        run: |
+          git submodule status
+          git submodule foreach 'git fetch --all --tags'
+          git submodule foreach 'git tag'
+
+      - name: 'Set up Python (${{ matrix.python.version }})'
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '${{ matrix.python.version }}'
+
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
+
+      - name: Initialize Pants and its GHA caches
+        uses: ./.github/actions/init-pants
+        with:
+          # To ignore a bad cache, bump the cache* integer.
+          gha-cache-key: cache0-py${{ matrix.python.version }}
+
+      - name: Export virtualenv to run Dev ST2 Cluster
+        run: >
+          pants
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          export
+          --resolve=st2
+
+      - name: Launch Dev ST2 Cluster
+        env:
+          VIRTUALENV_DIR: ./dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }}
+        run: |
+          sudo -E ./scripts/github/prepare-integration.sh
+
+      - name: Integration Tests
+        env:
+          ST2_CI_RUN_ORQUESTA_PAUSE_RESUME_TESTS: 'true'
+        run: >
+          pants
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          --tag=integration
+          --tag=st2cluster
+          test '::'
+
+      - name: Compress Service Logs Before upload
+        if: ${{ failure() }}
+        run: |
+          ./tools/launchdev.sh stop # stop st2 before collecting logs
+          tar cvzpf logs.tar.gz logs/*
+
+      - name: Upload StackStorm services Logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-py${{ matrix.python-version }}-st2cluster-integration-tests
+          path: logs.tar.gz
+          retention-days: 7
+
+      - name: Upload pants log
+        uses: actions/upload-artifact@v4
+        with:
+          name: pants-log-py${{ matrix.python.version }}-st2cluster-integration-tests
+          path: .pants.d/pants.log
+        if: always()  # We want the log even on failures.
 
   set_merge_ok:
     name: Set Merge OK (Tests)
@@ -321,7 +546,8 @@ jobs:
       - pants-plugins-tests
       - unit-tests
       - pack-tests
-      #- integration-tests
+      - integration-tests
+      - integration-st2cluster-tests
     outputs:
       merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }}
     runs-on: ubuntu-latest
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -69,7 +69,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
-  #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283
+  #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11
diff --git a/pants.toml b/pants.toml
@@ -257,6 +257,9 @@ extra_env_vars = [
   # "ST2_COORDINATION__URL", # Tests will override this with one of:
   #         "redis://{ST2TESTS_REDIS_HOST}:{ST2TESTS_REDIS_PORT}?namespace=_st2_test{ST2TESTS_PARALLEL_SLOT}
   #         "zake://"
+  # CI-specific vars
+  "ST2_CI",
+  "ST2_CI_RUN_ORQUESTA_PAUSE_RESUME_TESTS",
 ]
 # 10 min should be more than enough even for integration tests.
 timeout_default = 600 # seconds
