vector/.github/workflows/exaforce-publish.yaml at d6401efe8ecdfb8b7a62333931487b99382d4ce1 · ExaForce/vector · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
name: Publish

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
  push:
    branches:
      - "**-exaforce"
  workflow_dispatch:

# These permissions are needed to interact with GitHub's OIDC Token endpoint.
permissions:
  id-token: write
  contents: read

env:
  PUSH: ${{ github.event_name != 'pull_request' }}
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  VERBOSE: true
  CI: true
  DISABLE_MOLD: true
  DEBIAN_FRONTEND: noninteractive
  CONTAINER_TOOL: docker
  CARGO_PROFILE_RELEASE_LTO: fat
  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
  # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
  # can be removed when we switch back to the upstream openssl-sys crate
  CARGO_NET_GIT_FETCH_WITH_CLI: true

jobs:
  generate-publish-metadata:
    name: Generate Publish-related Metadata
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    outputs:
      vector_version: ${{ steps.generate-publish-metadata.outputs.vector_version }}
      vector_build_desc: ${{ steps.generate-publish-metadata.outputs.vector_build_desc }}
    steps:
      - name: Checkout Vector
        uses: actions/checkout@v4
      - name: Generate publish metadata
        id: generate-publish-metadata
        run: make ci-generate-publish-metadata

  build-x86_64-unknown-linux-gnu-packages:
    name: Build Vector for x86_64-unknown-linux-gnu (.tar.gz, DEB, RPM)
    runs-on: ubuntu-24.04-xl
    needs: generate-publish-metadata
    timeout-minutes: 90
    env:
      VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }}
      VECTOR_BUILD_DESC: ${{ needs.generate-publish-metadata.outputs.vector_build_desc }}
    steps:
      - name: Checkout Vector
        uses: actions/checkout@v4
      - name: Bootstrap runner environment (Ubuntu-specific)
        run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh
      - name: Bootstrap runner environment (generic)
        run: bash scripts/environment/prepare.sh
      - name: Build Vector
        run: |
          make test-integration-azure ENVIRONMENT=true
          make package-x86_64-unknown-linux-gnu-all ENVIRONMENT=true
      - name: Stage package artifacts for publish
        uses: actions/upload-artifact@v4
        with:
          name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu
          path: target/artifacts/vector*

  build-aarch64-unknown-linux-gnu-packages:
    name: Build Vector for aarch64-unknown-linux-gnu (.tar.gz)
    runs-on: ubuntu-24.04-xl
    timeout-minutes: 180
    needs: generate-publish-metadata
    env:
      VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }}
      VECTOR_BUILD_DESC: ${{ needs.generate-publish-metadata.outputs.vector_build_desc }}
    steps:
      - name: Checkout Vector
        uses: actions/checkout@v4
      - name: Bootstrap runner environment (Ubuntu-specific)
        run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh
      - name: Bootstrap runner environment (generic)
        run: bash scripts/environment/prepare.sh
      - name: Build Vector
        env:
          DOCKER_PRIVILEGED: "true"
        run: |
          make test-integration-azure ENVIRONMENT=true
          make package-aarch64-unknown-linux-gnu-all
      - name: Stage package artifacts for publish
        uses: actions/upload-artifact@v4
        with:
          name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu
          path: target/artifacts/vector*

  # deb-verify:
  #   name: Verify DEB Packages
  #   runs-on: ubuntu-20.04
  #   timeout-minutes: 5
  #   needs:
  #     - generate-publish-metadata
  #     - build-x86_64-unknown-linux-gnu-packages
  #   env:
  #     VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }}
  #     DD_PKG_VERSION: "latest"
  #   strategy:
  #     matrix:
  #       container:
  #         - ubuntu:20.04
  #         - ubuntu:22.04
  #   container:
  #     image: ${{ matrix.container }}
  #   steps:
  #     - run: |
  #         apt-get update && \
  #         apt-get install -y \
  #         ca-certificates \
  #         curl \
  #         git \
  #         systemd \
  #         make
  #     - name: Install dd-pkg for linting
  #       run: |
  #         curl -sSL "https://dd-package-tools.s3.amazonaws.com/dd-pkg/${DD_PKG_VERSION}/dd-pkg_Linux_x86_64.tar.gz" | tar -xz -C /usr/local/bin dd-pkg
  #     - name: Fix Git safe directories issue when in containers (actions/checkout#760)
  #       run: git config --global --add safe.directory /__w/vector/vector
  #     - name: Checkout Vector
  #       uses: actions/checkout@v4
  #       # with:
  #       #   ref: ${{ inputs.git_ref }}
  #     - name: Download staged package artifacts (x86_64-unknown-linux-gnu)
  #       uses: actions/download-artifact@v4
  #       with:
  #         name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu
  #         path: target/artifacts
  #     - name: Verify install of DEB package.
  #       run: |
  #         ./scripts/verify-install.sh target/artifacts/vector_${{ env.VECTOR_VERSION }}-1_amd64.deb

  publish-docker:
    name: Publish to Docker
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    needs:
      - generate-publish-metadata
      - build-x86_64-unknown-linux-gnu-packages
      - build-aarch64-unknown-linux-gnu-packages
      # - deb-verify
    env:
      VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }}
      VECTOR_BUILD_DESC: ${{ needs.generate-publish-metadata.outputs.vector_build_desc }}
    steps:
      - name: Checkout Vector
        uses: actions/checkout@v4
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::157206707803:role/github-actions-ExaForce-operations
          aws-region: us-east-2
          mask-aws-account-id: "false"
      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2
        with:
          mask-password: "false"
      - name: Set up QEMU
        uses: docker/[email protected]
        with:
          platforms: all
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/[email protected]
        with:
          version: latest
          install: true
      - name: Download staged package artifacts (x86_64-unknown-linux-gnu)
        uses: actions/download-artifact@v4
        with:
          name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu
          path: target/artifacts
      - name: Download staged package artifacts (aarch64-unknown-linux-gnu)
        uses: actions/download-artifact@v4
        with:
          name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu
          path: target/artifacts
      - name: Set SHA env
        run: echo "GITHUB_SHA_SHORT=$(echo ${{ github.event.pull_request.head.sha || github.sha }} | cut -c 1-8)" >> $GITHUB_ENV
      - name: Sanitize branch name
        run: echo "SANITIZED_BRANCH_NAME=$(echo ${{ github.ref_name }} | tr '/' '_')" >> $GITHUB_ENV
      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: target/artifacts
          file: distribution/docker/debian/Dockerfile
          push: ${{ env.PUSH }}
          tags: 157206707803.dkr.ecr.us-east-2.amazonaws.com/vector-base:${{ env.SANITIZED_BRANCH_NAME }}-${{ env.GITHUB_SHA_SHORT }} , 157206707803.dkr.ecr.us-east-2.amazonaws.com/vector-base:${{ env.SANITIZED_BRANCH_NAME }}-latest
          platforms: "linux/amd64,linux/arm64"