Skip to content

Commit 73a5a32

Browse files
dkropachevclaude
dkropachev
and
claude
committed
Optimize CI/CD pipeline: share build artifacts, eliminate redundant compilation
- Build job installs all modules and uploads .m2/repository/com/scylladb/ as artifacts for downstream jobs to reuse - Downstream jobs (verify, unit-tests, integration-tests) download pre-built artifacts and run `make install-all` offline to populate target/ directories without recompilation - Integration test jobs now set CI=true and MAVEN_OFFLINE_FLAG=-o to use cached artifacts and avoid network fetches - Makefile: add CI-aware variables (GUAVA_SHADED_DEP, INSTALL_ALL_DEP, MAVEN_IT_PL_ARGS) to skip redundant steps when artifacts are pre-built - Makefile: add install-all and check targets to separate fast install from full verification (fmt, clirr, animal-sniffer) - Remove compile-all dependency from download-all-dependencies since install-all already covers compile-scoped deps - Extract validate-build-artifacts into reusable composite action - Remove guava-shaded:jar dependency from integration-tests/pom.xml (handled by install-all in both CI and local builds) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 51b0a52 commit 73a5a32

4 files changed

Lines changed: 193 additions & 64 deletions

File tree

‎.github/workflows/docs-pr.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,4 @@ jobs:
4141
run: make -C docs setupenv
4242

4343
- name: Build docs
44-
run: make -C docs test
44+
run: make -C docs test

‎.github/workflows/tests@v1.yml‎

Lines changed: 153 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -27,40 +27,45 @@ on:
2727
- ".gitignore"
2828
workflow_dispatch:
2929

30+
permissions:
31+
contents: read
32+
checks: write
33+
3034
jobs:
3135
build:
3236
name: Build
3337
runs-on: ubuntu-latest
3438
timeout-minutes: 10
3539

36-
strategy:
37-
matrix:
38-
java-version: [8]
39-
fail-fast: false
40+
env:
41+
JAVA_VERSION: '8'
4042

4143
steps:
4244
- name: Checkout source
4345
uses: actions/checkout@v5
4446

45-
- name: Set up JDK ${{ matrix.java-version }}
47+
- name: Set up JDK ${{ env.JAVA_VERSION }}
4648
uses: actions/setup-java@v5
4749
with:
48-
java-version: ${{ matrix.java-version }}
50+
java-version: ${{ env.JAVA_VERSION }}
4951
distribution: 'temurin'
5052

51-
- name: Get POM hash
52-
id: get-pom-hash
53-
run: echo "value=${{ hashFiles('**/pom.xml') }}" >> "$GITHUB_OUTPUT"
54-
5553
- name: Restore maven repository cache
5654
uses: actions/cache/restore@v4
5755
id: java-cache
5856
with:
5957
path: ~/.m2/repository
60-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }}
58+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
59+
restore-keys: |
60+
${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-
6161
62-
- name: Compile source and tests
63-
run: make compile-all
62+
- name: Create settings-security.xml
63+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
64+
65+
- name: Install all modules
66+
run: make install-all
67+
env:
68+
MAVEN_OFFLINE_FLAG: ''
6469

6570
- name: Download test dependencies
6671
if: steps.java-cache.outputs.cache-hit != 'true'
@@ -71,62 +76,124 @@ jobs:
7176
if: steps.java-cache.outputs.cache-hit != 'true'
7277
with:
7378
path: ~/.m2/repository
74-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }}
79+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
80+
81+
- name: Upload build targets
82+
uses: actions/upload-artifact@v4
83+
with:
84+
name: build-targets
85+
path: |
86+
*/target/
87+
*/*/target/
88+
retention-days: 3
89+
90+
- name: Upload scylladb JARs
91+
uses: actions/upload-artifact@v4
92+
with:
93+
name: scylladb-jars
94+
path: ~/.m2/repository/com/scylladb/
95+
retention-days: 3
7596

7697
verify:
7798
name: Full verify
7899
runs-on: ubuntu-latest
100+
needs: [build]
79101
timeout-minutes: 10
80102

81-
strategy:
82-
matrix:
83-
java-version: [8]
84-
fail-fast: false
103+
env:
104+
JAVA_VERSION: '8'
85105

86106
steps:
87107
- name: Checkout source
88108
uses: actions/checkout@v5
89109

90-
- name: Set up JDK ${{ matrix.java-version }}
110+
- name: Set up JDK ${{ env.JAVA_VERSION }}
91111
uses: actions/setup-java@v5
92112
with:
93-
java-version: ${{ matrix.java-version }}
113+
java-version: ${{ env.JAVA_VERSION }}
94114
distribution: 'temurin'
95115

96116
- name: Restore maven repository cache
97117
uses: actions/cache/restore@v4
118+
id: java-cache
98119
with:
99120
path: ~/.m2/repository
100-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
121+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
122+
123+
- name: Verify maven cache was restored
124+
if: steps.java-cache.outputs.cache-hit != 'true'
125+
run: |
126+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
127+
exit 1
128+
129+
- name: Download build targets
130+
uses: actions/download-artifact@v4
131+
with:
132+
name: build-targets
133+
134+
- name: Download scylladb JARs
135+
uses: actions/download-artifact@v4
136+
with:
137+
name: scylladb-jars
138+
path: ~/.m2/repository/com/scylladb/
139+
140+
- name: Touch build targets to prevent recompilation
141+
run: find . -path '*/target/*' -type f -exec touch {} +
142+
143+
- name: Create settings-security.xml
144+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
101145

102146
- name: Full verify
103147
run: make check
104148

105149
unit-tests:
106150
name: Unit tests
107151
runs-on: ubuntu-latest
152+
needs: [build]
108153
timeout-minutes: 10
109154

110-
strategy:
111-
matrix:
112-
java-version: [8]
113-
fail-fast: false
155+
env:
156+
JAVA_VERSION: '8'
114157

115158
steps:
116159
- name: Checkout source
117160
uses: actions/checkout@v5
118161

119-
- name: Set up JDK 8
162+
- name: Set up JDK ${{ env.JAVA_VERSION }}
120163
uses: actions/setup-java@v5
121164
with:
122-
java-version: ${{ matrix.java-version }}
165+
java-version: ${{ env.JAVA_VERSION }}
123166
distribution: 'temurin'
124167

125168
- name: Restore maven repository cache
126169
uses: actions/cache/restore@v4
170+
id: java-cache
127171
with:
128172
path: ~/.m2/repository
129-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
173+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
174+
175+
- name: Verify maven cache was restored
176+
if: steps.java-cache.outputs.cache-hit != 'true'
177+
run: |
178+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
179+
exit 1
180+
181+
- name: Download build targets
182+
uses: actions/download-artifact@v4
183+
with:
184+
name: build-targets
185+
186+
- name: Download scylladb JARs
187+
uses: actions/download-artifact@v4
188+
with:
189+
name: scylladb-jars
190+
path: ~/.m2/repository/com/scylladb/
191+
192+
- name: Touch build targets to prevent recompilation
193+
run: find . -path '*/target/*' -type f -exec touch {} +
194+
195+
- name: Create settings-security.xml
196+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
130197

131198
- name: Run unit tests
132199
run: make test-unit
@@ -135,8 +202,8 @@ jobs:
135202
if: always()
136203
run: |
137204
shopt -s globstar
138-
mkdir unit
139-
cp --parents ./**/target/*-reports/*.xml unit/
205+
mkdir -p unit
206+
cp --parents ./**/target/*-reports/*.xml unit/ || true
140207
141208
- name: Upload test results
142209
uses: actions/upload-artifact@v4
@@ -157,24 +224,10 @@ jobs:
157224
updateComment: false
158225
skip_annotations: true
159226

160-
setup-integration-tests:
161-
name: Setup ITs
162-
runs-on: ubuntu-latest
163-
timeout-minutes: 2
164-
165-
steps:
166-
- name: Checkout source
167-
uses: actions/checkout@v5
168-
169-
- name: Setup Python 3
170-
uses: actions/setup-python@v6
171-
with:
172-
python-version: '3.13'
173-
174227
cassandra-integration-tests:
175228
name: Cassandra ITs
176229
runs-on: ubuntu-latest
177-
needs: [setup-integration-tests]
230+
needs: [build]
178231
timeout-minutes: 90
179232

180233
strategy:
@@ -196,10 +249,34 @@ jobs:
196249

197250
- name: Restore maven repository cache
198251
uses: actions/cache/restore@v4
252+
id: java-cache
199253
with:
200254
path: ~/.m2/repository
201255
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
202256

257+
- name: Verify maven cache was restored
258+
if: steps.java-cache.outputs.cache-hit != 'true'
259+
run: |
260+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
261+
exit 1
262+
263+
- name: Download build targets
264+
uses: actions/download-artifact@v4
265+
with:
266+
name: build-targets
267+
268+
- name: Download scylladb JARs
269+
uses: actions/download-artifact@v4
270+
with:
271+
name: scylladb-jars
272+
path: ~/.m2/repository/com/scylladb/
273+
274+
- name: Touch build targets to prevent recompilation
275+
run: find . -path '*/target/*' -type f -exec touch {} +
276+
277+
- name: Create settings-security.xml
278+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
279+
203280
- name: Setup Python 3
204281
uses: actions/setup-python@v6
205282
with:
@@ -246,6 +323,7 @@ jobs:
246323
- name: Run integration tests on Cassandra (${{ steps.cassandra-version.outputs.value }}) - ${{ matrix.test-group }}
247324
id: run-integration-tests
248325
env:
326+
CI: true
249327
CASSANDRA_VERSION_RESOLVED: ${{ steps.cassandra-version.outputs.value }}
250328
MAVEN_EXTRA_ARGS: ${{ steps.test-skip-args.outputs.value }}
251329
run: make test-integration-cassandra
@@ -273,7 +351,7 @@ jobs:
273351

274352
- name: Parse test results
275353
uses: mikepenz/action-junit-report@v5
276-
if: always()
354+
if: steps.run-integration-tests.outcome != 'skipped'
277355
with:
278356
check_name: Integration tests report for Cassandra ${{ steps.cassandra-version.outputs.value }} (${{ matrix.test-group }})
279357
require_tests: true
@@ -286,7 +364,7 @@ jobs:
286364
scylla-integration-tests:
287365
name: Scylla ITs
288366
runs-on: ubuntu-latest
289-
needs: [setup-integration-tests]
367+
needs: [build]
290368
timeout-minutes: 90
291369

292370
strategy:
@@ -308,10 +386,34 @@ jobs:
308386

309387
- name: Restore maven repository cache
310388
uses: actions/cache/restore@v4
389+
id: java-cache
311390
with:
312391
path: ~/.m2/repository
313392
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
314393

394+
- name: Verify maven cache was restored
395+
if: steps.java-cache.outputs.cache-hit != 'true'
396+
run: |
397+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
398+
exit 1
399+
400+
- name: Download build targets
401+
uses: actions/download-artifact@v4
402+
with:
403+
name: build-targets
404+
405+
- name: Download scylladb JARs
406+
uses: actions/download-artifact@v4
407+
with:
408+
name: scylladb-jars
409+
path: ~/.m2/repository/com/scylladb/
410+
411+
- name: Touch build targets to prevent recompilation
412+
run: find . -path '*/target/*' -type f -exec touch {} +
413+
414+
- name: Create settings-security.xml
415+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
416+
315417
- name: Setup Python 3
316418
uses: actions/setup-python@v6
317419
with:
@@ -322,6 +424,8 @@ jobs:
322424

323425
- name: Get scylla version
324426
id: scylla-version
427+
env:
428+
SCYLLA_VERSION: ${{ matrix.scylla-version }}
325429
run: make resolve-scylla-version
326430

327431
- name: Pull CCM image from the cache
@@ -356,6 +460,7 @@ jobs:
356460
- name: Run integration tests on Scylla (${{ steps.scylla-version.outputs.value }}) - ${{ matrix.test-group }}
357461
id: run-integration-tests
358462
env:
463+
CI: true
359464
SCYLLA_VERSION_RESOLVED: ${{ steps.scylla-version.outputs.value }}
360465
MAVEN_EXTRA_ARGS: ${{ steps.test-skip-args.outputs.value }}
361466
run: make test-integration-scylla
@@ -383,7 +488,7 @@ jobs:
383488

384489
- name: Parse test results
385490
uses: mikepenz/action-junit-report@v5
386-
if: always()
491+
if: steps.run-integration-tests.outcome != 'skipped'
387492
with:
388493
check_name: Integration tests report for Scylla ${{ steps.scylla-version.outputs.value }} (${{ matrix.test-group }})
389494
require_tests: true

0 commit comments

Comments
 (0)