Skip to content

Commit 8635b36

Browse files
ohmayrohmayr
committed
update workflow
1 parent 4ea79e6 commit 8635b36

2 files changed

Lines changed: 111 additions & 177 deletions

File tree

‎.github/workflows/experiment.yaml‎

Lines changed: 70 additions & 177 deletions
Original file line numberDiff line numberDiff line change
@@ -1,156 +1,74 @@
1-
name: Next-Gen CI Pipeline
2-
3-
on:
4-
pull_request:
5-
branches: [ main, preview ]
6-
merge_group:
7-
types: [checks_requested]
8-
9-
concurrency:
10-
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
11-
cancel-in-progress: true
1+
name: CI Next Gen
2+
on: [pull_request]
123

134
jobs:
14-
# ==========================================
15-
# 1. DISCOVERY ENGINE (The Matrix Sharder)
16-
# ==========================================
5+
# =========================================================
6+
# 1. DISCOVERY (Pure Python, Zero External Files)
7+
# =========================================================
178
discover:
189
runs-on: ubuntu-latest
1910
outputs:
20-
chunks: ${{ steps.build-matrix.outputs.chunks }}
11+
chunks: ${{ steps.chunking.outputs.chunks }}
12+
python_versions: '["3.9", "3.10", "3.11", "3.12", "3.14"]'
2113
steps:
2214
- uses: actions/checkout@v4
23-
with:
24-
fetch-depth: 0
25-
26-
- name: Detect Changed Packages
27-
id: changes
15+
16+
- id: changes
2817
uses: tj-actions/changed-files@v44
29-
with:
30-
files: packages/**
31-
dir_names: true
32-
dir_names_max_depth: 2
18+
# Note: Ensure you configure this to only grab package directories
3319

34-
- name: Build Dynamic Shards
35-
id: build-matrix
20+
- id: chunking
3621
env:
37-
RAW_LIST: ${{ steps.changes.outputs.all_changed_files }}
22+
CHANGED_FILES: ${{ steps.changes.outputs.all_changed_files }}
3823
run: |
3924
python -c '
4025
import os, json
41-
packages = os.environ.get("RAW_LIST", "").split()
42-
print(f"Detected {len(packages)} changed packages.")
4326
44-
if not packages:
45-
chunks = []
46-
else:
47-
# Shard into groups of 10 to completely bypass GitHubs 256 total job limit
48-
chunk_size = 10
49-
chunks = [" ".join(packages[i:i+chunk_size]) for i in range(0, len(packages), chunk_size)]
27+
raw_files = os.environ.get("CHANGED_FILES", "")
28+
packages = raw_files.split() if raw_files else []
29+
30+
# Hardcap at 20 buckets to mathematically guarantee limit safety
31+
NUM_BUCKETS = 20
32+
chunks = [[] for _ in range(NUM_BUCKETS)]
33+
for i, pkg in enumerate(packages):
34+
chunks[i % NUM_BUCKETS].append(pkg)
35+
36+
formatted = [" ".join(chunk) for chunk in chunks if chunk]
5037
51-
print(f"Created {len(chunks)} matrix shards.")
5238
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
53-
f.write(f"chunks={json.dumps(chunks)}\n")
39+
f.write(f"chunks={json.dumps(formatted)}\n")
5440
'
5541
56-
# ==========================================
57-
# 2. STATIC ANALYSIS
58-
# ==========================================
59-
static-checks:
60-
needs: discover
61-
if: ${{ needs.discover.outputs.chunks != '[]' }}
62-
runs-on: ubuntu-latest
63-
strategy:
64-
fail-fast: false
65-
matrix:
66-
chunk: ${{ fromJSON(needs.discover.outputs.chunks) }}
67-
steps:
68-
- uses: actions/checkout@v4
69-
- uses: astral-sh/setup-uv@v5
70-
with:
71-
python-version: "3.14"
72-
enable-cache: true
73-
- name: Run Lint and MyPy
74-
run: |
75-
export NOX_DEFAULT_VENV_BACKEND=uv
76-
export UV_VENV_SEED=1
77-
78-
FAILED=0
79-
for pkg in ${{ matrix.chunk }}; do
80-
echo "::group::Linting $pkg"
81-
cd $pkg
82-
if ! uvx --with 'nox[uv]' nox -s lint mypy lint_setup_py; then
83-
echo "::error::Linting failed in $pkg"
84-
FAILED=1
85-
fi
86-
cd $GITHUB_WORKSPACE
87-
echo "::endgroup::"
88-
done
89-
exit $FAILED
90-
91-
# ==========================================
92-
# 3. DOCUMENTATION BUILD
93-
# ==========================================
94-
docs-build:
42+
# =========================================================
43+
# 2. UNIT TESTS (Max 100 Jobs, 50 Concurrent)
44+
# =========================================================
45+
unit-tests:
9546
needs: discover
9647
if: ${{ needs.discover.outputs.chunks != '[]' }}
9748
runs-on: ubuntu-latest
9849
strategy:
9950
fail-fast: false
51+
max-parallel: 50
10052
matrix:
10153
chunk: ${{ fromJSON(needs.discover.outputs.chunks) }}
54+
python-version: ${{ fromJSON(needs.discover.outputs.python_versions) }}
55+
56+
name: Unit (Chunk, ${{ matrix.python-version }})
10257
steps:
10358
- uses: actions/checkout@v4
10459
- uses: astral-sh/setup-uv@v5
10560
with:
106-
python-version: "3.10"
61+
python-version: ${{ matrix.python-version }}
10762
enable-cache: true
108-
- name: Build Docs and DocFX
109-
run: |
110-
export NOX_DEFAULT_VENV_BACKEND=uv
111-
export UV_VENV_SEED=1
11263

113-
FAILED=0
114-
for pkg in ${{ matrix.chunk }}; do
115-
echo "::group::Docs $pkg"
116-
cd $pkg
117-
if ! uvx --with 'nox[uv]' nox -s docs docfx; then
118-
echo "::error::Docs failed in $pkg"
119-
FAILED=1
120-
fi
121-
cd $GITHUB_WORKSPACE
122-
echo "::endgroup::"
123-
done
124-
exit $FAILED
125-
126-
# ==========================================
127-
# 4a. UNIT TESTS (Python 3.9)
128-
# ==========================================
129-
unit-tests-3-9:
130-
needs: discover
131-
if: ${{ needs.discover.outputs.chunks != '[]' }}
132-
runs-on: ubuntu-latest
133-
strategy:
134-
fail-fast: false
135-
matrix:
136-
chunk: ${{ fromJSON(needs.discover.outputs.chunks) }}
137-
steps:
138-
- uses: actions/checkout@v4
139-
- uses: astral-sh/setup-uv@v5
140-
with:
141-
python-version: "3.9"
142-
enable-cache: true
143-
- name: Execute Unit Tests (Python 3.9)
64+
- name: Run Unit Tests
14465
run: |
14566
export NOX_DEFAULT_VENV_BACKEND=uv
146-
export UV_VENV_SEED=1
147-
export UV_COMPILE_BYTECODE=1
148-
14967
FAILED=0
15068
for pkg in ${{ matrix.chunk }}; do
15169
echo "::group::Testing $pkg"
15270
cd $pkg
153-
if ! uvx --with 'nox[uv]' nox -s unit-3.9; then
71+
if ! uvx --with 'nox[uv]' nox -s "unit-${{ matrix.python-version }}"; then
15472
echo "::error::Failed in $pkg"
15573
FAILED=1
15674
fi
@@ -159,113 +77,88 @@ jobs:
15977
done
16078
exit $FAILED
16179
162-
# ==========================================
163-
# 4b. UNIT TESTS (Python 3.14)
164-
# ==========================================
165-
unit-tests-3-14:
80+
# =========================================================
81+
# 3. LINT & DOCS (Max 20 Jobs, 20 Concurrent)
82+
# =========================================================
83+
lint-and-docs:
16684
needs: discover
16785
if: ${{ needs.discover.outputs.chunks != '[]' }}
16886
runs-on: ubuntu-latest
16987
strategy:
17088
fail-fast: false
89+
max-parallel: 20
17190
matrix:
17291
chunk: ${{ fromJSON(needs.discover.outputs.chunks) }}
92+
93+
name: Lint & Docs (Chunk)
17394
steps:
17495
- uses: actions/checkout@v4
17596
- uses: astral-sh/setup-uv@v5
17697
with:
177-
python-version: "3.14"
98+
python-version: "3.14" # Linting/Docs usually only require latest Python
17899
enable-cache: true
179-
- name: Execute Unit Tests (Python 3.14)
100+
101+
- name: Run Lint and Docs
180102
run: |
181103
export NOX_DEFAULT_VENV_BACKEND=uv
182-
export UV_VENV_SEED=1
183-
export UV_COMPILE_BYTECODE=1
184-
185104
FAILED=0
186105
for pkg in ${{ matrix.chunk }}; do
187-
echo "::group::Testing $pkg"
106+
echo "::group::Linting & Docs for $pkg"
188107
cd $pkg
189-
if ! uvx --with 'nox[uv]' nox -s unit-3.14; then
190-
echo "::error::Failed in $pkg"
191-
FAILED=1
108+
109+
# Look Before You Leap: Run lint if it exists in the package
110+
if uvx --with 'nox[uv]' nox -l | grep -q "lint"; then
111+
uvx --with 'nox[uv]' nox -s "lint" || FAILED=1
192112
fi
113+
114+
# Look Before You Leap: Run docs if it exists
115+
if uvx --with 'nox[uv]' nox -l | grep -q "docs"; then
116+
uvx --with 'nox[uv]' nox -s "docs" || FAILED=1
117+
fi
118+
193119
cd $GITHUB_WORKSPACE
194120
echo "::endgroup::"
195121
done
196122
exit $FAILED
197123
198-
# ==========================================
199-
# 5. CORE DEPS FROM SOURCE (Pure UV)
200-
# ==========================================
201-
unit-core-deps-from-source:
124+
# =========================================================
125+
# 4. CORE DEPENDENCIES (Max 100 Jobs, 50 Concurrent)
126+
# =========================================================
127+
core-deps:
202128
needs: discover
203129
if: ${{ needs.discover.outputs.chunks != '[]' }}
204130
runs-on: ubuntu-latest
205131
strategy:
206132
fail-fast: false
133+
max-parallel: 50
207134
matrix:
208135
chunk: ${{ fromJSON(needs.discover.outputs.chunks) }}
136+
python-version: ${{ fromJSON(needs.discover.outputs.python_versions) }}
137+
138+
name: Core Deps (Chunk, ${{ matrix.python-version }})
209139
steps:
210140
- uses: actions/checkout@v4
211141
- uses: astral-sh/setup-uv@v5
212142
with:
213-
python-version: "3.14"
143+
python-version: ${{ matrix.python-version }}
214144
enable-cache: true
215-
- name: Execute Core Deps
145+
146+
- name: Run Core Deps
216147
run: |
217148
export NOX_DEFAULT_VENV_BACKEND=uv
218-
export UV_VENV_SEED=1
219-
export UV_COMPILE_BYTECODE=1
220-
221-
# NATIVELY instruct uv to allow prerelease Git resolution
222-
export UV_PRERELEASE=allow
223-
224149
FAILED=0
225150
for pkg in ${{ matrix.chunk }}; do
226-
echo "::group::Testing $pkg"
151+
echo "::group::Core Deps Testing $pkg"
227152
cd $pkg
228153
229-
# The Look-Before-You-Leap check
230154
if uvx --with 'nox[uv]' nox -l | grep -q "core_deps_from_source"; then
231-
# Natively executes with pure UV speed using your newly patched Noxfiles
232-
if ! uvx --with 'nox[uv]' nox -s "core_deps_from_source" -p "3.14"; then
233-
echo "::error::Failed in $pkg"
234-
FAILED=1
235-
fi
155+
# Pass the explicit Python version down to Nox
156+
uvx --with 'nox[uv]' nox -s "core_deps_from_source" -p "${{ matrix.python-version }}" || FAILED=1
236157
else
237-
echo "Skipping: 'core_deps_from_source' is not defined for $pkg."
158+
echo "Skipping: 'core_deps_from_source' not defined in $pkg."
238159
fi
239160
240161
cd $GITHUB_WORKSPACE
241162
echo "::endgroup::"
242163
done
243164
exit $FAILED
244-
245-
# ==========================================
246-
# 6. THE GATEKEEPER
247-
# ==========================================
248-
presubmit-passed:
249-
if: always()
250-
needs:
251-
- discover
252-
- static-checks
253-
- docs-build
254-
- unit-tests-3-9
255-
- unit-tests-3-14
256-
- unit-core-deps-from-source
257-
runs-on: ubuntu-latest
258-
steps:
259-
- name: Evaluate Pipeline Status
260-
run: |
261-
if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
262-
echo "::error::One or more required CI jobs failed."
263-
exit 1
264-
fi
265-
266-
if [[ "${{ needs.discover.outputs.chunks }}" == "[]" ]]; then
267-
echo "No packages changed. Safely bypassing execution."
268-
exit 0
269-
fi
270-
271-
echo "All jobs passed."

‎uv_reinstall.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import os
2+
import re
3+
from pathlib import Path
4+
5+
def patch_noxfiles():
6+
updated = 0
7+
# Walk through all packages in the monorepo
8+
for path in Path("packages").rglob("noxfile.py"):
9+
content = path.read_text(encoding="utf-8")
10+
11+
# Skip files that don't implement the session or don't use the legacy flag
12+
if "def core_deps_from_source" not in content or '"--ignore-installed"' not in content:
13+
continue
14+
15+
# Regex captures the exact whitespace/indentation before the target loop
16+
pattern = re.compile(
17+
r'^([ \t]*)for dep in core_dependencies_from_source:\n[ \t]*session\.install\(dep, "--no-deps", "--ignore-installed"\)',
18+
re.MULTILINE
19+
)
20+
21+
def replacer(match):
22+
indent = match.group(1)
23+
# Inject the backend-aware variable and apply it to the loop
24+
return (
25+
f'{indent}# Natively adapt the overwrite flag based on the active resolver\n'
26+
f'{indent}force_overwrite_flag = "--reinstall" if os.environ.get("NOX_DEFAULT_VENV_BACKEND") == "uv" else "--ignore-installed"\n'
27+
f'{indent}for dep in core_dependencies_from_source:\n'
28+
f'{indent} session.install(dep, "--no-deps", force_overwrite_flag)'
29+
)
30+
31+
new_content, count = pattern.subn(replacer, content)
32+
33+
if count > 0:
34+
path.write_text(new_content, encoding="utf-8")
35+
print(f"✅ Patched: {path}")
36+
updated += 1
37+
38+
print(f"\nSuccess: Safely patched {updated} noxfiles.")
39+
40+
if __name__ == "__main__":
41+
patch_noxfiles()

0 commit comments

Comments
 (0)