Skip to content

Commit e838a59

Browse files
dkropachevdkropachev
committed
Optimize CI/CD pipeline: share build artifacts, eliminate redundant compilation
- Build job now runs `mvn install` and uploads SNAPSHOT artifacts for downstream jobs to download, avoiding full recompilation in every job - All jobs (verify, unit-tests, ITs) depend on build and reuse its artifacts - Remove the empty setup-integration-tests job - Disable Maven -X debug logging in CI to reduce log noise - Use $(MVNCMD) consistently across all Makefile targets - Skip guava-shaded recompilation in compile-all/test-unit via -pl exclusion - In CI, restrict IT reactor to integration-tests module only (-pl flag) - Suppress settings-security.xml FileNotFoundException warning in all jobs
1 parent b12f886 commit e838a59

2 files changed

Lines changed: 77 additions & 30 deletions

File tree

‎.github/workflows/tests@v1.yml‎

Lines changed: 52 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,11 @@ jobs:
5959
path: ~/.m2/repository
6060
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }}
6161

62-
- name: Compile source and tests
63-
run: make compile-all
62+
- name: Create settings-security.xml
63+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
64+
65+
- name: Install all modules
66+
run: make install-all
6467

6568
- name: Download test dependencies
6669
if: steps.java-cache.outputs.cache-hit != 'true'
@@ -73,9 +76,17 @@ jobs:
7376
path: ~/.m2/repository
7477
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }}
7578

79+
- name: Upload build artifacts
80+
uses: actions/upload-artifact@v4
81+
with:
82+
name: build-artifacts
83+
path: ~/.m2/repository/com/scylladb/
84+
retention-days: 1
85+
7686
verify:
7787
name: Full verify
7888
runs-on: ubuntu-latest
89+
needs: [build]
7990
timeout-minutes: 10
8091

8192
strategy:
@@ -99,12 +110,22 @@ jobs:
99110
path: ~/.m2/repository
100111
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
101112

113+
- name: Download build artifacts
114+
uses: actions/download-artifact@v4
115+
with:
116+
name: build-artifacts
117+
path: ~/.m2/repository/com/scylladb/
118+
119+
- name: Create settings-security.xml
120+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
121+
102122
- name: Full verify
103123
run: make check
104124

105125
unit-tests:
106126
name: Unit tests
107127
runs-on: ubuntu-latest
128+
needs: [build]
108129
timeout-minutes: 10
109130

110131
strategy:
@@ -128,6 +149,15 @@ jobs:
128149
path: ~/.m2/repository
129150
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
130151

152+
- name: Download build artifacts
153+
uses: actions/download-artifact@v4
154+
with:
155+
name: build-artifacts
156+
path: ~/.m2/repository/com/scylladb/
157+
158+
- name: Create settings-security.xml
159+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
160+
131161
- name: Run unit tests
132162
run: make test-unit
133163

@@ -157,24 +187,10 @@ jobs:
157187
updateComment: false
158188
skip_annotations: true
159189

160-
setup-integration-tests:
161-
name: Setup ITs
162-
runs-on: ubuntu-latest
163-
timeout-minutes: 2
164-
165-
steps:
166-
- name: Checkout source
167-
uses: actions/checkout@v5
168-
169-
- name: Setup Python 3
170-
uses: actions/setup-python@v6
171-
with:
172-
python-version: '3.13'
173-
174190
cassandra-integration-tests:
175191
name: Cassandra ITs
176192
runs-on: ubuntu-latest
177-
needs: [setup-integration-tests]
193+
needs: [build]
178194
timeout-minutes: 90
179195

180196
strategy:
@@ -200,6 +216,15 @@ jobs:
200216
path: ~/.m2/repository
201217
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
202218

219+
- name: Download build artifacts
220+
uses: actions/download-artifact@v4
221+
with:
222+
name: build-artifacts
223+
path: ~/.m2/repository/com/scylladb/
224+
225+
- name: Create settings-security.xml
226+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
227+
203228
- name: Setup Python 3
204229
uses: actions/setup-python@v6
205230
with:
@@ -286,7 +311,7 @@ jobs:
286311
scylla-integration-tests:
287312
name: Scylla ITs
288313
runs-on: ubuntu-latest
289-
needs: [setup-integration-tests]
314+
needs: [build]
290315
timeout-minutes: 90
291316

292317
strategy:
@@ -312,6 +337,15 @@ jobs:
312337
path: ~/.m2/repository
313338
key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }}
314339

340+
- name: Download build artifacts
341+
uses: actions/download-artifact@v4
342+
with:
343+
name: build-artifacts
344+
path: ~/.m2/repository/com/scylladb/
345+
346+
- name: Create settings-security.xml
347+
run: echo '<settingsSecurity/>' > ~/.m2/settings-security.xml
348+
315349
- name: Setup Python 3
316350
uses: actions/setup-python@v6
317351
with:

‎Makefile‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,20 @@ CCM_SCYLLA_REPO ?= github.com/scylladb/scylla-ccm
1212
CCM_SCYLLA_VERSION ?= master
1313

1414
SCYLLA_EXT_OPTS ?= --smp=2 --memory=4G
15-
MVNCMD ?= mvn -B -X -ntp
15+
ifndef CI
16+
MAVEN_DEBUG_FLAG ?= -X
17+
endif
18+
MVNCMD ?= mvn -B $(MAVEN_DEBUG_FLAG) -ntp
19+
20+
# In CI, build artifacts are pre-installed; skip guava-shaded install and
21+
# restrict IT reactor to integration-tests module only.
22+
ifdef CI
23+
GUAVA_SHADED_DEP :=
24+
MAVEN_IT_PL_ARGS ?= -pl integration-tests
25+
else
26+
GUAVA_SHADED_DEP := .install-guava-shaded
27+
MAVEN_IT_PL_ARGS ?=
28+
endif
1629

1730
GET_VERSION_VERSION ?= 0.4.3
1831

@@ -36,9 +49,6 @@ export PATH := $(MAKEFILE_PATH)/bin:$(PATH)
3649
.install-guava-shaded:
3750
$(MVNCMD) install -pl guava-shaded
3851

39-
.install-all-modules:
40-
$(MVNCMD) install -DskipTests -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
41-
4252
.download-test-dependencies:
4353
$(MVNCMD) test -Dtest=TestThatDoesNotExists -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true || true
4454

@@ -241,37 +251,40 @@ release-dry-run: .require-release-env
241251
mkdir /tmp/java-driver-release-logs/ 2>/dev/null || true
242252
$(MVNCMD) release:perform > >(tee /tmp/java-driver-release-logs/stdout.log) 2> >(tee /tmp/java-driver-release-logs/stderr.log)
243253

244-
compile-all: .install-guava-shaded
245-
mvn -B compile test-compile -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
254+
install-all:
255+
$(MVNCMD) install -DskipTests -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
256+
257+
compile-all: $(GUAVA_SHADED_DEP)
258+
$(MVNCMD) compile test-compile -pl '!guava-shaded' -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
246259

247260
check:
248261
$(MVNCMD) verify -DskipTests
249262

250263
fix:
251264
$(MVNCMD) fmt:format
252265

253-
test-unit: .install-guava-shaded
254-
$(MVNCMD) test -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
266+
test-unit: $(GUAVA_SHADED_DEP)
267+
$(MVNCMD) test -pl '!guava-shaded' -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true
255268

256-
test-integration-scylla: .install-all-modules .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr
269+
test-integration-scylla: $(GUAVA_SHADED_DEP) .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr
257270
@if [[ -z "$${SCYLLA_VERSION_RESOLVED}" ]]; then
258271
SCYLLA_VERSION_RESOLVED=`cat '${SCYLLA_VERSION_FILE}'`
259272
fi
260273
if [[ -z "$${SCYLLA_VERSION_RESOLVED}" ]]; then
261274
echo "ScyllaDB version ${SCYLLA_VERSION} was not resolved"
262275
exit 1
263276
fi
264-
mvn -B -e verify -pl integration-tests -Dccm.version=$${SCYLLA_VERSION_RESOLVED} -Dccm.distribution=scylla -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS)
277+
$(MVNCMD) verify $(MAVEN_IT_PL_ARGS) -Dccm.version=$${SCYLLA_VERSION_RESOLVED} -Dccm.distribution=scylla -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS)
265278

266-
test-integration-cassandra: .install-all-modules .prepare-scylla-ccm resolve-cassandra-version
279+
test-integration-cassandra: $(GUAVA_SHADED_DEP) .prepare-scylla-ccm resolve-cassandra-version
267280
@if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then
268281
CASSANDRA_VERSION_RESOLVED=`cat '${CASSANDRA_VERSION_FILE}'`
269282
fi
270283
if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then
271284
echo "Cassandra version ${CASSANDRA_VERSION} was not resolved"
272285
exit 1
273286
fi
274-
mvn -B -e verify -pl integration-tests -Dccm.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS)
287+
$(MVNCMD) verify $(MAVEN_IT_PL_ARGS) -Dccm.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS)
275288

276289
check-no-compile-warnings:
277290
@$(MAKE) compile-all | grep WARNING >/tmp/all-compile-warnings.log || true

0 commit comments

Comments
 (0)