Skip to content

Commit 3328c7c

Browse files
dkropachevclaude
dkropachev
and
claude
committed
Optimize CI/CD pipeline: share build artifacts, eliminate redundant compilation
- Build job installs all modules and uploads .m2/repository/com/scylladb/ as artifacts for downstream jobs to reuse - Downstream jobs (verify, unit-tests, integration-tests) download pre-built artifacts and run `make install-all` offline to populate target/ directories without recompilation - Integration test jobs now set CI=true and MAVEN_OFFLINE_FLAG=-o to use cached artifacts and avoid network fetches - Makefile: add CI-aware variables (GUAVA_SHADED_DEP, INSTALL_ALL_DEP, MAVEN_IT_PL_ARGS) to skip redundant steps when artifacts are pre-built - Makefile: add install-all and check targets to separate fast install from full verification (fmt, clirr, animal-sniffer) - Remove compile-all dependency from download-all-dependencies since install-all already covers compile-scoped deps - Extract validate-build-artifacts into reusable composite action - Remove guava-shaded:jar dependency from integration-tests/pom.xml (handled by install-all in both CI and local builds) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 51b0a52 commit 3328c7c

4 files changed

Lines changed: 196 additions & 64 deletions

File tree

‎.github/workflows/docs-pr.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,4 @@ jobs:
4141
run: make -C docs setupenv
4242

4343
- name: Build docs
44-
run: make -C docs test
44+
run: make -C docs test

‎.github/workflows/tests@v1.yml‎

Lines changed: 156 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -27,106 +27,178 @@ on:
2727
- ".gitignore"
2828
workflow_dispatch:
2929

30+
permissions:
31+
contents: read
32+
checks: write
33+
34+
env:
35+
CI: true
36+
3037
jobs:
3138
build:
3239
name: Build
3340
runs-on: ubuntu-latest
3441
timeout-minutes: 10
3542

36-
strategy:
37-
matrix:
38-
java-version: [8]
39-
fail-fast: false
43+
env:
44+
JAVA_VERSION: '8'
4045

4146
steps:
4247
- name: Checkout source
4348
uses: actions/checkout@v5
4449

45-
- name: Set up JDK ${{ matrix.java-version }}
50+
- name: Set up JDK ${{ env.JAVA_VERSION }}
4651
uses: actions/setup-java@v5
4752
with:
48-
java-version: ${{ matrix.java-version }}
53+
java-version: ${{ env.JAVA_VERSION }}
4954
distribution: 'temurin'
5055

51-
- name: Get POM hash
52-
id: get-pom-hash
53-
run: echo "value=${{ hashFiles('**/pom.xml') }}" >> "$GITHUB_OUTPUT"
54-
5556
- name: Restore maven repository cache
5657
uses: actions/cache/restore@v4
5758
id: java-cache
5859
with:
5960
path: ~/.m2/repository
60-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }}
61+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
62+
restore-keys: |
63+
${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-
6164
62-
- name: Compile source and tests
63-
run: make compile-all
65+
- name: Create settings-security.xml
66+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
67+
68+
- name: Install all modules
69+
run: make install-all
70+
env:
71+
MAVEN_OFFLINE_FLAG: ''
6472

6573
- name: Download test dependencies
6674
if: steps.java-cache.outputs.cache-hit != 'true'
6775
run: make download-all-dependencies
76+
env:
77+
MAVEN_OFFLINE_FLAG: '' # Override CI offline mode to allow fetching dependencies
6878

6979
- name: Save maven repository cache
7080
uses: actions/cache/save@v4
7181
if: steps.java-cache.outputs.cache-hit != 'true'
7282
with:
7383
path: ~/.m2/repository
74-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }}
84+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
85+
86+
- name: Upload build targets
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: build-targets
90+
path: |
91+
*/target/
92+
*/*/target/
93+
retention-days: 3
94+
95+
- name: Upload scylladb JARs
96+
uses: actions/upload-artifact@v4
97+
with:
98+
name: scylladb-jars
99+
path: ~/.m2/repository/com/scylladb/
100+
retention-days: 3
75101

76102
verify:
77103
name: Full verify
78104
runs-on: ubuntu-latest
105+
needs: [build]
79106
timeout-minutes: 10
80107

81-
strategy:
82-
matrix:
83-
java-version: [8]
84-
fail-fast: false
108+
env:
109+
JAVA_VERSION: '8'
85110

86111
steps:
87112
- name: Checkout source
88113
uses: actions/checkout@v5
89114

90-
- name: Set up JDK ${{ matrix.java-version }}
115+
- name: Set up JDK ${{ env.JAVA_VERSION }}
91116
uses: actions/setup-java@v5
92117
with:
93-
java-version: ${{ matrix.java-version }}
118+
java-version: ${{ env.JAVA_VERSION }}
94119
distribution: 'temurin'
95120

96121
- name: Restore maven repository cache
97122
uses: actions/cache/restore@v4
123+
id: java-cache
98124
with:
99125
path: ~/.m2/repository
100-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
126+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
127+
128+
- name: Verify maven cache was restored
129+
if: steps.java-cache.outputs.cache-hit != 'true'
130+
run: |
131+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
132+
exit 1
133+
134+
- name: Download build targets
135+
uses: actions/download-artifact@v4
136+
with:
137+
name: build-targets
138+
139+
- name: Download scylladb JARs
140+
uses: actions/download-artifact@v4
141+
with:
142+
name: scylladb-jars
143+
path: ~/.m2/repository/com/scylladb/
144+
145+
- name: Touch build targets to prevent recompilation
146+
run: find . -path '*/target/*' -type f -exec touch {} +
147+
148+
- name: Create settings-security.xml
149+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
101150

102151
- name: Full verify
103152
run: make check
104153

105154
unit-tests:
106155
name: Unit tests
107156
runs-on: ubuntu-latest
157+
needs: [build]
108158
timeout-minutes: 10
109159

110-
strategy:
111-
matrix:
112-
java-version: [8]
113-
fail-fast: false
160+
env:
161+
JAVA_VERSION: '8'
114162

115163
steps:
116164
- name: Checkout source
117165
uses: actions/checkout@v5
118166

119-
- name: Set up JDK 8
167+
- name: Set up JDK ${{ env.JAVA_VERSION }}
120168
uses: actions/setup-java@v5
121169
with:
122-
java-version: ${{ matrix.java-version }}
170+
java-version: ${{ env.JAVA_VERSION }}
123171
distribution: 'temurin'
124172

125173
- name: Restore maven repository cache
126174
uses: actions/cache/restore@v4
175+
id: java-cache
127176
with:
128177
path: ~/.m2/repository
129-
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
178+
key: ${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }}
179+
180+
- name: Verify maven cache was restored
181+
if: steps.java-cache.outputs.cache-hit != 'true'
182+
run: |
183+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
184+
exit 1
185+
186+
- name: Download build targets
187+
uses: actions/download-artifact@v4
188+
with:
189+
name: build-targets
190+
191+
- name: Download scylladb JARs
192+
uses: actions/download-artifact@v4
193+
with:
194+
name: scylladb-jars
195+
path: ~/.m2/repository/com/scylladb/
196+
197+
- name: Touch build targets to prevent recompilation
198+
run: find . -path '*/target/*' -type f -exec touch {} +
199+
200+
- name: Create settings-security.xml
201+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
130202

131203
- name: Run unit tests
132204
run: make test-unit
@@ -135,8 +207,8 @@ jobs:
135207
if: always()
136208
run: |
137209
shopt -s globstar
138-
mkdir unit
139-
cp --parents ./**/target/*-reports/*.xml unit/
210+
mkdir -p unit
211+
cp --parents ./**/target/*-reports/*.xml unit/ || true
140212
141213
- name: Upload test results
142214
uses: actions/upload-artifact@v4
@@ -157,24 +229,10 @@ jobs:
157229
updateComment: false
158230
skip_annotations: true
159231

160-
setup-integration-tests:
161-
name: Setup ITs
162-
runs-on: ubuntu-latest
163-
timeout-minutes: 2
164-
165-
steps:
166-
- name: Checkout source
167-
uses: actions/checkout@v5
168-
169-
- name: Setup Python 3
170-
uses: actions/setup-python@v6
171-
with:
172-
python-version: '3.13'
173-
174232
cassandra-integration-tests:
175233
name: Cassandra ITs
176234
runs-on: ubuntu-latest
177-
needs: [setup-integration-tests]
235+
needs: [build]
178236
timeout-minutes: 90
179237

180238
strategy:
@@ -196,10 +254,34 @@ jobs:
196254

197255
- name: Restore maven repository cache
198256
uses: actions/cache/restore@v4
257+
id: java-cache
199258
with:
200259
path: ~/.m2/repository
201260
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
202261

262+
- name: Verify maven cache was restored
263+
if: steps.java-cache.outputs.cache-hit != 'true'
264+
run: |
265+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
266+
exit 1
267+
268+
- name: Download build targets
269+
uses: actions/download-artifact@v4
270+
with:
271+
name: build-targets
272+
273+
- name: Download scylladb JARs
274+
uses: actions/download-artifact@v4
275+
with:
276+
name: scylladb-jars
277+
path: ~/.m2/repository/com/scylladb/
278+
279+
- name: Touch build targets to prevent recompilation
280+
run: find . -path '*/target/*' -type f -exec touch {} +
281+
282+
- name: Create settings-security.xml
283+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
284+
203285
- name: Setup Python 3
204286
uses: actions/setup-python@v6
205287
with:
@@ -273,7 +355,7 @@ jobs:
273355

274356
- name: Parse test results
275357
uses: mikepenz/action-junit-report@v5
276-
if: always()
358+
if: steps.run-integration-tests.outcome != 'skipped'
277359
with:
278360
check_name: Integration tests report for Cassandra ${{ steps.cassandra-version.outputs.value }} (${{ matrix.test-group }})
279361
require_tests: true
@@ -286,7 +368,7 @@ jobs:
286368
scylla-integration-tests:
287369
name: Scylla ITs
288370
runs-on: ubuntu-latest
289-
needs: [setup-integration-tests]
371+
needs: [build]
290372
timeout-minutes: 90
291373

292374
strategy:
@@ -308,10 +390,34 @@ jobs:
308390

309391
- name: Restore maven repository cache
310392
uses: actions/cache/restore@v4
393+
id: java-cache
311394
with:
312395
path: ~/.m2/repository
313396
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
314397

398+
- name: Verify maven cache was restored
399+
if: steps.java-cache.outputs.cache-hit != 'true'
400+
run: |
401+
echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache."
402+
exit 1
403+
404+
- name: Download build targets
405+
uses: actions/download-artifact@v4
406+
with:
407+
name: build-targets
408+
409+
- name: Download scylladb JARs
410+
uses: actions/download-artifact@v4
411+
with:
412+
name: scylladb-jars
413+
path: ~/.m2/repository/com/scylladb/
414+
415+
- name: Touch build targets to prevent recompilation
416+
run: find . -path '*/target/*' -type f -exec touch {} +
417+
418+
- name: Create settings-security.xml
419+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
420+
315421
- name: Setup Python 3
316422
uses: actions/setup-python@v6
317423
with:
@@ -322,6 +428,8 @@ jobs:
322428

323429
- name: Get scylla version
324430
id: scylla-version
431+
env:
432+
SCYLLA_VERSION: ${{ matrix.scylla-version }}
325433
run: make resolve-scylla-version
326434

327435
- name: Pull CCM image from the cache
@@ -383,7 +491,7 @@ jobs:
383491

384492
- name: Parse test results
385493
uses: mikepenz/action-junit-report@v5
386-
if: always()
494+
if: steps.run-integration-tests.outcome != 'skipped'
387495
with:
388496
check_name: Integration tests report for Scylla ${{ steps.scylla-version.outputs.value }} (${{ matrix.test-group }})
389497
require_tests: true

0 commit comments

Comments
 (0)