From fdca8e7990756484156191c8a5bdb4b68165eff6 Mon Sep 17 00:00:00 2001 From: njukenanli Date: Sun, 31 May 2026 16:11:28 +0800 Subject: [PATCH] implement windows dockerfile generator --- .github/workflows/tests.yml | 13 + docs/Development.md | 4 +- launch/scripts/gen_dockerfile.py | 125 ++++++++-- launch/scripts/recollect.py | 81 ------ tests/gen_dockerfile_test.py | 407 +++++++++++++++++++++++++++++++ tests/runtime_test.py | 4 +- 6 files changed, 528 insertions(+), 106 deletions(-) delete mode 100644 launch/scripts/recollect.py create mode 100644 tests/gen_dockerfile_test.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7efa600..47dcf70 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -71,6 +71,19 @@ jobs: docker version docker info + - name: Install Docker Buildx + # The windows-2025 runner ships Docker Engine but not the buildx CLI plugin, which + # test_gen_dockerfile_syntax needs for `docker buildx build --check`. Install the + # plugin binary so it uses the implicit 'default' builder (the host docker daemon); + # we deliberately do NOT create a docker-container builder (that driver is Linux-only). + run: | + $version = "v0.34.1" + $url = "https://github.com/docker/buildx/releases/download/$version/buildx-$version.windows-amd64.exe" + $dest = Join-Path $env:ProgramData "Docker\cli-plugins" + New-Item -ItemType Directory -Force -Path $dest | Out-Null + Invoke-WebRequest -Uri $url -OutFile (Join-Path $dest "docker-buildx.exe") + docker buildx version + - name: Install test dependencies run: | python -m pip install --upgrade pip diff --git a/docs/Development.md b/docs/Development.md index 47d68e1..29ba19e 100644 --- a/docs/Development.md +++ b/docs/Development.md @@ -205,11 +205,11 @@ python -m launch.scripts.upload_docker\ --clear_after_push 0 # 0 for false and 1 for true ``` -### Re-assemble Dockerfile (Beta / Preview) +### Re-assemble Dockerfile Reconstruct Dockerfile of a commited image from instance["docker_image_layers"]. -Note this script is still under development. If you find any bugs of this script, welcome GitHub issues and pull requests. +The Dockerfile behavior strictly aligns with that of RepoLaunch-created images. It produces two layers (the setup layer and the organize layer) with error commands silenty bypassed instead of interuptting the build. ```bash python -m launch.scripts.gen_dockerfile \ diff --git a/launch/scripts/gen_dockerfile.py b/launch/scripts/gen_dockerfile.py index 760a243..c6779ab 100644 --- a/launch/scripts/gen_dockerfile.py +++ b/launch/scripts/gen_dockerfile.py @@ -18,6 +18,11 @@ import json import os from typing import Any, Literal, Optional, TypedDict +import logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) class LayerInfo(TypedDict): base_image: str @@ -29,9 +34,22 @@ class LayerInfo(TypedDict): LINUX_WORKDIR = "/testbed" WINDOWS_WORKDIR = r"C:\testbed" -# Heredoc / here-string delimiters chosen to be unlikely to appear in any real command. +# Heredoc delimiter for the linux generator (BuildKit heredoc works on linux). LINUX_HEREDOC_TAG = "RL_CMD_EOF" -WINDOWS_HEREDOC_TAG = "RL_CMD_EOF" + +# Windows sentinels. The windows generator must work on the *legacy* Docker builder +# (Docker Desktop in Windows-container mode does not use BuildKit, and no +# `docker/dockerfile` frontend tag publishes a Windows manifest, so heredocs are +# unavailable for windows containers). The legacy builder also strips literal double +# quotes from shell-form RUN instructions. So every command is carried as a +# single-quoted PowerShell string with these substitutions, then reconstituted into a +# .ps1 at build time: +# " -> WINDOWS_DQ_SENTINEL (decoded to [char]34 ; avoids the builder eating quotes) +# \n -> WINDOWS_NL_SENTINEL (decoded to [char]10 ; avoids RUN line-splitting) +# ' -> '' (PowerShell single-quoted-string escaping) +# Chosen to be extremely unlikely to appear verbatim in any real command. +WINDOWS_DQ_SENTINEL = "~~RLDQ~~" +WINDOWS_NL_SENTINEL = "~~RLNL~~" def _render_linux_layer(commands: list[str], comment: str) -> list[str]: @@ -75,31 +93,90 @@ def gen_linux_dockerfile(layers: LayerInfo) -> str: return "\n".join(lines) -def _render_windows_layer(commands: list[str], comment: str) -> list[str]: +def _windows_layer_script_path(comment: str) -> str: + """Stable, unlikely-to-collide path for the per-layer script staged into the image.""" + slug = comment.strip().lower().replace(" ", "_") + return rf"C:\rl_{slug}.ps1" + + +def _encode_windows_command(cmd: str) -> str: """ - Render one windows layer as a single Dockerfile RUN that uses a BuildKit heredoc - feeding a literal here-string into PowerShell. + Encode one command for transport inside a single-quoted PowerShell string on one + physical (backtick-continued) Dockerfile line. See WINDOWS_*_SENTINEL above. - Each input command is wrapped in a try/catch so multi-line commands stay readable - and one failure does not abort the build (note 5). Single quotes inside the - `@'...'@` here-string are doubled per PowerShell rules. + Substitution order is significant: hide double quotes first (plain swap), then + escape single quotes for the surrounding single-quoted string, then hide newlines. + The two sentinels are disjoint from `'`-doubling so decode order at build time does + not matter. + """ + s = cmd.rstrip("\n") + s = s.replace('"', WINDOWS_DQ_SENTINEL) + s = s.replace("'", "''") + s = s.replace("\r\n", "\n").replace("\n", WINDOWS_NL_SENTINEL) + return s + + +def _render_windows_layer(commands: list[str], comment: str) -> list[str]: + """ + Render one windows layer as a single RUN instruction (one layer per setup/organize, + note 3/4) that assembles a .ps1 inside the image and executes it. + + The RUN does, in order (each step its own backtick-continued physical line): + - Set-Content an empty .ps1, then one Add-Content per command appending its + encoded `try { } catch { ... }` block (note 5: a failure -- PowerShell + error -- is swallowed; a nonzero native exit code does not throw, so execution + simply falls through to the next command); + - decode the two sentinels back to real `"` and newlines, rewriting the .ps1 as a + normal multi-line script (note 6: multi-line commands are preserved verbatim); + - execute the .ps1. + + Why not a heredoc: Docker Desktop builds windows containers with the *legacy* + builder (no BuildKit), and no `docker/dockerfile` frontend image is published for + windows, so `RUN < str: @@ -107,8 +184,11 @@ def gen_windows_dockerfile(layers: LayerInfo) -> str: setup_cmds: list[str] = list(layers.get("setup_layer") or []) organize_cmds: list[str] = list(layers.get("organize_layer") or []) + # `# escape=`` switches the line-continuation char to a backtick so each layer's RUN + # can span multiple physical lines (one Add-Content per line). No `# syntax` + # directive: it would force pulling the dockerfile frontend image, which is not + # published for windows and fails to resolve. lines: list[str] = [ - "# syntax=docker/dockerfile:1.4", "# escape=`", f"FROM {base_image}", f"WORKDIR {WINDOWS_WORKDIR}", @@ -121,6 +201,9 @@ def gen_windows_dockerfile(layers: LayerInfo) -> str: def main(instances: list[dict[str, Any]], output_dir: Path, platform: Literal["linux", "windows"]) -> None: + logging.info(("The gen_dockerfile script produces a Dockerfile from the command sequence of RepoLaunch. ", + "The Dockerfile behavior strictly aligns with that of RepoLaunch-created images: " + "it produces two layers (the setup layer and the organize layer) with error commands silenty bypassed instead of interuptting the build.\n")) for instance in instances: filename: str = "Dockerfile_" + instance["instance_id"].strip().replace("/", "_") + "_" + platform filepath: Path = (output_dir / filename) diff --git a/launch/scripts/recollect.py b/launch/scripts/recollect.py deleted file mode 100644 index efa6611..0000000 --- a/launch/scripts/recollect.py +++ /dev/null @@ -1,81 +0,0 @@ - -import json -from pathlib import Path -from typing import Literal -from launch.core.runtime import SetupRuntime -from fire import Fire -from launch.scripts.parser import run_get_pertest_cmd, run_parser - -def main(workspace: str, platform: Literal["linux", "windows"]): - ''' - workspace: the place that stores setup.jsonl, organize.jsonl, playground... - ''' - workspace = Path(workspace) - playground = workspace / "playground" - output_jsonl = workspace / f"organize.jsonl" - swe_instances = [] - max_len = 5000_0000 - all_len = 0 - for subfolder in playground.iterdir(): - if not subfolder.is_dir(): - continue - - instance_path = subfolder / "instance.json" - result_path = subfolder / "result.json" - - if not instance_path.exists() or not result_path.exists(): - continue - - instance = json.loads(instance_path.read_text()) - result = json.loads(result_path.read_text()) - - if not result.get("organize_completed", False): - continue - - if not result.get("unittest_generator", ""): - continue - - try: - container = SetupRuntime.from_launch_image(result["docker_image"], result["instance_id"], platform) - except: - print("pulling image timeout, skipping") - continue - test_output = container.send_command(";".join(result["test_commands"])).output # unstripped / full result - test_status: dict[str, str] = run_parser(result["log_parser"], test_output) - test_list = [i for i in test_status.keys() if test_status[i] == "pass"] - pertest_cmd: dict[str, str] = run_get_pertest_cmd(result["unittest_generator"], test_list) - # for debug - print(test_status) - print(pertest_cmd, flush = True) - container.cleanup() - - if not pertest_cmd: - continue - - swe_instance = { - **instance, - "setup_cmds": result.get("setup_commands", []), - "test_cmds": result["test_commands"], - "print_cmds": result.get("print_commands", []), - "log_parser": result.get("log_parser", "pytest"), - "docker_image": result.get("docker_image", f"karinali20011210/migbench:{instance["instance_id"]}_{platform}"), - } - swe_instance["rebuild_cmds"] = result["rebuild_commands"] - swe_instance["test_status"] = test_status - swe_instance["pertest_command"] = pertest_cmd - swe_instance["log_parser"] = result["log_parser"] - swe_instance["per_test_command_generator"] = result["unittest_generator"] - - swe_instances.append(swe_instance) - - all_len += len(str(pertest_cmd)) - if all_len > max_len: - break - - with open(output_jsonl, "w") as f: - for i in swe_instances: - json.dump(i, f) - f.write("\n") - -if __name__ == "__main__": - Fire(main) \ No newline at end of file diff --git a/tests/gen_dockerfile_test.py b/tests/gen_dockerfile_test.py new file mode 100644 index 0000000..bad56cc --- /dev/null +++ b/tests/gen_dockerfile_test.py @@ -0,0 +1,407 @@ +MOCK_PLATFORM_INSTANCES = { + "windows" : [ +r"""{ + "instance_id": "dotnet__runtime-126064", + "docker_image_layers": { + "base_image": "mcr.microsoft.com/dotnet/sdk:8.0-windowsservercore-ltsc2022", + "setup_layer": [ + "\n # Skip if git already present\n if (-not (Get-Command git.exe -ErrorAction SilentlyContinue)) {\n try {\n # Prefer Chocolatey (cleaner package mgmt)\n if (-not (Get-Command choco.exe -ErrorAction SilentlyContinue)) {\n Set-ExecutionPolicy Bypass -Scope Process -Force\n [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12\n Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))\n }\n\n choco install git.install -y --no-progress --params '\"/GitOnlyOnPath /NoAutoCrlf\"'\n }\n catch {\n Write-Host \"Chocolatey install failed: $($_.Exception.Message) -> falling back to Git for Windows installer\"\n\n # Fallback: Official Git for Windows silent install\n $ProgressPreference = 'SilentlyContinue'\n $temp = Join-Path $env:TEMP 'git-installer.exe'\n # 'latest' link maintained by Git for Windows; resolves to current amd64 EXE\n $url = 'https://github.com/git-for-windows/git/releases/latest/download/Git-64-bit.exe'\n try {\n Invoke-WebRequest -Uri $url -OutFile $temp\n } catch {\n Invoke-WebRequest -UseBasicParsing -Uri $url -OutFile $temp\n }\n\n # Silent/unattended flags per Git for Windows docs (Inno Setup):\n # /VERYSILENT /NORESTART /NOCANCEL /SP- /CLOSEAPPLICATIONS /RESTARTAPPLICATIONS\n # Optional components: icons, ext\\reg\\shellhere, assoc, assoc_sh, gitlfs, windowsterminal, scalar\n Start-Process -FilePath $temp -ArgumentList `\n '/VERYSILENT','/NORESTART','/NOCANCEL','/SP-','/CLOSEAPPLICATIONS','/RESTARTAPPLICATIONS',`\n '/COMPONENTS=\"icons,ext\\reg\\shellhere,assoc,assoc_sh,gitlfs,windowsterminal,scalar\"' `\n -Wait\n }\n\n # Ensure PATH is updated in this running session (Chocolatey/Git installers update registry only)\n $gitCmd = 'C:\\Program Files\\Git\\cmd'\n $gitBin = 'C:\\Program Files\\Git\\bin'\n if (Test-Path $gitCmd) { $env:PATH = \"$gitCmd;$gitBin;$env:PATH\" }\n }\n ", + "git config --global --add safe.directory \"C:\\testbed\"; git init \"C:\\testbed\"; cd \"C:\\testbed\"; git remote add origin https://github.com/dotnet/runtime.git; git fetch --depth 1 origin d69ff06d522242b57825def7bb613fda6d4beebb; git reset --hard d69ff06d522242b57825def7bb613fda6d4beebb", + "ls", + "Get-ChildItem -Recurse -Filter *.sln | Select-Object -First 20", + "Get-ChildItem -Recurse -Filter *.csproj | Select-Object -First 20", + "Get-Content src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj", + "dotnet restore src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj", + "Get-Content global.json", + "@'\n{\n \"sdk\": {\n \"version\": \"8.0.420\",\n \"allowPrerelease\": false,\n \"rollForward\": \"latestFeature\"\n },\n \"tools\": {\n \"dotnet\": \"8.0.420\"\n },\n \"msbuild-sdks\": {\n \"Microsoft.DotNet.Arcade.Sdk\": \"11.0.0-beta.26172.108\",\n \"Microsoft.DotNet.Helix.Sdk\": \"11.0.0-beta.26172.108\",\n \"Microsoft.DotNet.SharedFramework.Sdk\": \"11.0.0-beta.26172.108\",\n \"Microsoft.Build.NoTargets\": \"3.7.0\",\n \"Microsoft.Build.Traversal\": \"3.4.0\",\n \"Microsoft.NET.Sdk.IL\": \"11.0.0-preview.3.26172.108\"\n }\n}\n'@ | Set-Content -Path global.json", + "dotnet restore src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj", + "dotnet build src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release", + "Select-String -Path Directory.Build.props,Directory.Build.targets,eng\\*.props,eng\\*.targets -Pattern \"GenerateResxSource|Arcade\" -SimpleMatch", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\tools\\GenerateResxSource.targets", + "dotnet build src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:GenerateResxSource=false", + "Select-String -Path . -Filter *.props,*.targets -Recurse -Pattern \"GenerateResxSource\" -SimpleMatch", + "Get-ChildItem -Recurse -Include *.props,*.targets | Select-String -Pattern \"GenerateResxSource\" -SimpleMatch", + "Get-Content Directory.Build.props", + "Get-Content Directory.Build.rsp", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108 -Recurse -Include *.props,*.targets | Select-String -Pattern \"Disable|GenerateResxSource|ArcadeBuildTasks|BuildTasks\" -SimpleMatch", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\Sdks\\Microsoft.DotNet.Arcade.Sdk\\Sdk.targets", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\sdk", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\sdk\\Sdk.targets", + "Get-Content Directory.Build.targets", + "Get-Content eng\\resources.targets", + "Get-Content src\\tools\\illink\\src\\ILLink.RoslynAnalyzer\\ILLink.RoslynAnalyzer.csproj", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108 -Recurse -Include *.targets,*.props | Select-String -Pattern \"_GenerateResxSource|GenerateResxSource\" -SimpleMatch", + "Get-Content src\\coreclr\\tools\\aot\\DependencyGraphViewer\\DependencyGraphViewer.csproj", + "Get-ChildItem -Recurse -Filter SR.cs | Select-String -Pattern \"System.Reflection.Metadata.SR\" -SimpleMatch", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\tools\\Imports.targets", + "Get-Content src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj", + "Get-Content eng\\generatorProjects.targets", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\sdk\\Sdk.props", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\tools\\GenerateResxSource.targets", + "dotnet build src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:GenerateResxSource=false", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj,*.sln,*.proj | Select-String -Pattern \"ILLink.RoslynAnalyzer\" -SimpleMatch", + "Get-Content eng\\liveBuilds.targets", + "Get-Content eng\\generators.targets", + "Get-Content Directory.Build.props", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\tools", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108\\tools\\net", + "Get-Content eng\\liveILLink.targets", + "dotnet build src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj | Select-String -Pattern \"Microsoft.Net.Compilers.Toolset\" -SimpleMatch", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj | Select-String -Pattern \"Compilers.Toolset|Net.Compilers\" -SimpleMatch", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.net.compilers.toolset\\5.6.0-2.26172.108\\build\\Microsoft.Net.Compilers.Toolset.props", + "Get-Content Directory.Build.rsp", + "Get-Content src\\coreclr\\tools\\aot\\DependencyGraphViewer\\DependencyGraphViewer.csproj", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj,*.sln | Select-String -Pattern \"Net.Compilers.Toolset\" -SimpleMatch", + "Get-ChildItem \"C:\\Program Files\\dotnet\\packs\"", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj | Select-String -Pattern \"DisableImplicitFrameworkReferences|DisableStandardFrameworkReferences\" -SimpleMatch", + "Get-ChildItem -Recurse -Include Directory.Packages.props,*.props,*.targets,*.csproj | Select-String -Pattern \"Microsoft.Net.Compilers.Toolset\" -SimpleMatch", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj,*.sln,*.json | Select-String -Pattern \"5.6.0-2.26172.108|Microsoft.Net.Compilers.Toolset\" -SimpleMatch", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj,*.sln,*.json | Select-String -Pattern \"CompilersToolset\" -SimpleMatch", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 /p:RoslynCompilerType=Framework", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj | Select-String -Pattern \"Roslyn|CompilerType|NetCompilersToolset\" -SimpleMatch", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108 -Recurse -Include *.props,*.targets | Select-String -Pattern \"RoslynCompilerType|NetCompilersToolset\" -SimpleMatch", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj | Select-String -Pattern \"MicrosoftNetCompilersToolset\" -SimpleMatch", + "Get-Content Directory.Build.props", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.net.compilers.toolset\\5.6.0-2.26172.108\\build\\Microsoft.Net.Compilers.Toolset.targets", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.net.compilers.toolset\\5.6.0-2.26172.108\\build", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj,*.sln,*.json | Select-String -Pattern \"CompilersToolset\" -SimpleMatch", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.dotnet.arcade.sdk\\11.0.0-beta.26172.108 -Recurse -Include *.props,*.targets | Select-String -Pattern \"Net.Compilers.Toolset|CompilersToolset\" -SimpleMatch", + "Get-Content artifacts\\obj\\coreclr\\DependencyGraphViewer\\DependencyGraphViewer.csproj.nuget.g.props", + "Get-ChildItem -Recurse -Filter Directory.Packages.props", + "Get-ChildItem C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.net.compilers.toolset\\5.6.0-2.26172.108 -Recurse -Include *.props,*.targets | Select-String -Pattern \"UseNetCompilersToolset|Disable|Toolset\" -SimpleMatch", + "Get-Content C:\\Users\\ContainerAdministrator\\.nuget\\packages\\microsoft.net.compilers.toolset\\5.6.0-2.26172.108\\build\\Microsoft.Net.Compilers.Toolset.props", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj,*.sln | Select-String -Pattern \"MicrosoftNetCompilersToolsetVersion\" -SimpleMatch", + "Get-ChildItem -Recurse -Include *.props,*.targets,*.csproj,*.sln | Select-String -Pattern \"Compilers\" -SimpleMatch", + "Get-Content eng\\Versions.props", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 --logger \"console;verbosity=detailed\"" + ], + "organize_layer": [ + "dotnet build src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0", + "New-Item -ItemType Directory -Force -Path reports | Out-Null; dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 --logger \"json;LogFileName=reports\\test-results.json\"", + "New-Item -ItemType Directory -Force -Path reports | Out-Null; dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 --logger \"trx;LogFileName=reports\\test-results.trx\"", + "Get-Content src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\TestResults\\reports\\test-results.trx", + "Get-Content src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\TestResults\\reports\\test-results.trx", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 --filter \"FullyQualifiedName=DependecyGraphViewer.Tests.TestFiileParsing.DependsOn\" --logger \"trx;LogFileName=reports\\single.trx\"", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 --list-tests", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 --filter \"FullyQualifiedName=DependecyGraphViewer.Tests.TestFileParsing.DependsOn\" --logger \"trx;LogFileName=reports\\single.trx\"", + "dotnet test src\\coreclr\\tools\\aot\\DependencyGraphViewer\\Tests\\DependecyGraphViewer.Tests.csproj -c Release /p:_RequiresLiveILLink=false /p:UsingToolMicrosoftNetCompilers=false /p:RunAnalyzers=false /p:EnableNETAnalyzers=false /p:NetCoreAppToolCurrentVersion=8.0 /p:NetCoreAppCurrentVersion=8.0 /p:NetCoreAppToolCurrent=net8.0 /p:NetCoreAppCurrent=net8.0 --filter \"FullyQualifiedName=DependecyGraphViewer.Tests.TestFileParsing.NumberOfNodes&DisplayName~nodeCount: 0, isValid: False, linkCount: 0\" --logger \"trx;LogFileName=reports\\single.trx\"" + ] + } +}""", +# A synthetic instance whose commands fail in every way (a throwing cmdlet, a native +# nonzero exit, and a failing *last* command in the *last* layer). A correct generator +# must still build the image successfully end-to-end (note 5). +r"""{ + "instance_id": "synthetic__windows-failing", + "docker_image_layers": { + "base_image": "mcr.microsoft.com/dotnet/sdk:8.0-windowsservercore-ltsc2022", + "setup_layer": [ + "Write-Host marker-setup-ran", + "Get-Item C:\\does\\not\\exist", + "cmd /c exit 7" + ], + "organize_layer": [ + "Write-Host marker-organize-ran", + "cmd /c exit 9" + ] + } +}""" +], + +"android": [ +r"""{ + "instance_id": "android-security-lints-5c27", + "docker_image_layers": { + "base_image": "cimg/android:2026.03.1", + "setup_layer": [ + "command -v git >/dev/null || (apt-get update && apt-get install -y git)", + "git config --global --add safe.directory /testbed; git init /testbed; cd /testbed; git remote add origin https://github.com/google/android-security-lints.git; git fetch --depth 1 origin 5c27e1bcfb29f55020e1529d3d18e476813ef79e; git reset --hard 5c27e1bcfb29f55020e1529d3d18e476813ef79e", + "./gradlew test ", + "./gradlew test ", + "./gradlew test --info", + "./gradlew :checks:test --rerun-tasks", + "for f in checks/build/test-results/test/TEST-*.xml; do echo \"==== $f\"; cat \"$f\"; done" + ], + "organize_layer": [ + "./gradlew build", + "./gradlew assemble", + "./gradlew :checks:assemble", + "./gradlew :checks:test --rerun-tasks", + "for f in checks/build/test-results/test/TEST-*.xml; do echo \"==== $f\"; cat \"$f\"; done", + "./gradlew build", + "./gradlew assemble", + "./gradlew :checks:assemble", + "./gradlew :checks:test --rerun-tasks", + "for f in checks/build/test-results/test/TEST-*.xml; do echo \"==== $f\"; cat \"$f\"; done", + "./gradlew :checks:test --tests \"com.example.lint.checks.BadCryptographyUsageDetectorTest.testWhenNoUnsafeAlgoUsed_noWarning\" --rerun-tasks", + "pwd", + "ls checks/src/test/java/com/example/lint/checks", + "./gradlew build", + "./gradlew assemble", + "./gradlew :checks:assemble", + "./gradlew :checks:test --rerun-tasks", + "for f in checks/build/test-results/test/TEST-*.xml; do echo \"==== $f\"; cat \"$f\"; done", + "./gradlew build", + "./gradlew assemble", + "./gradlew :checks:assemble", + "./gradlew :checks:test --rerun-tasks", + "for f in checks/build/test-results/test/TEST-*.xml; do echo \"==== $f\"; cat \"$f\"; done", + "./gradlew :checks:test --tests \"com.example.lint.checks.BadCryptographyUsageDetectorTest.testWhenNoUnsafeAlgoUsed_noWarning\" --rerun-tasks", + "pwd", + "ls checks/src/test/java/com/example/lint/checks" + ] + } +}""", +# A synthetic instance whose commands fail in every way (a command that errors, a +# nonzero exit, and a failing *last* command in the *last* layer). A correct generator +# must still build the image successfully end-to-end (note 5). +r"""{ + "instance_id": "synthetic__android-failing", + "docker_image_layers": { + "base_image": "cimg/android:2026.03.1", + "setup_layer": [ + "echo marker-setup-ran", + "false", + "this-command-does-not-exist" + ], + "organize_layer": [ + "echo marker-organize-ran", + "exit 3" + ] + } +}""" +], + +"linux": [ +r"""{ + "instance_id": "oxc-project__oxc-21163", + "docker_image_layers": { + "base_image": "rust:1.90", + "setup_layer": [ + "apt update && apt install -y git", + "git config --global --add safe.directory /testbed; git init /testbed; cd /testbed; git remote add origin https://github.com/oxc-project/oxc.git; git fetch --depth 1 origin 8e2ed83efd88806872d9ebb48960e3018fbea9c2; git reset --hard 8e2ed83efd88806872d9ebb48960e3018fbea9c2", + "rustup toolchain install 1.94.1", + "rustup override set 1.94.1", + "cargo ck", + "apt-get update && apt-get install -y cmake", + "cargo ck", + "cargo test --all-features", + "sed -n '160,240p' crates/oxc_codegen/tests/integration/sourcemap.rs", + "apt-get update && apt-get install -y nodejs", + "curl -fsSL https://nodejs.org/dist/v24.14.0/node-v24.14.0-linux-x64.tar.xz -o /tmp/node.tar.xz && tar -C /usr/local --strip-components=1 -xJf /tmp/node.tar.xz", + "node --version && cargo test --all-features", + "npm install -g oxlint-tsgolint@0.20.0", + "cargo test --all-features -- --nocapture" + ], + "organize_layer": [ + "cd /testbed && npm --version && node --version && cargo --version", + "cd /testbed && test -f package-lock.json && echo has-lock || echo no-lock", + "cd /testbed && ls -1 package-lock.json 2>/dev/null || true", + "cd /testbed && mkdir -p reports && cargo test --all-features -- --format json 2>&1 | tee reports/cargo-test.jsonl", + "cd /testbed && mkdir -p reports && cargo test --all-features -- --nocapture 2>&1 | tee reports/cargo-test.log", + "cd /testbed && cat reports/cargo-test.log", + "cd /testbed && cargo test -p oxc_allocator --all-features allocator::test::string_from_empty_array -- --nocapture" + ] + } +}""", +# A synthetic instance whose commands fail in every way (a command that errors, a +# nonzero exit, and a failing *last* command in the *last* layer). A correct generator +# must still build the image successfully end-to-end (note 5). +r"""{ + "instance_id": "synthetic__linux-failing", + "docker_image_layers": { + "base_image": "rust:1.90", + "setup_layer": [ + "echo marker-setup-ran", + "false", + "this-command-does-not-exist" + ], + "organize_layer": [ + "echo marker-organize-ran", + "exit 3" + ] + } +}""" +] +} + +import json +import platform +import subprocess +import uuid +from pathlib import Path +from typing import Any + +import pytest + +from launch.scripts.gen_dockerfile import ( + _encode_windows_command, + gen_linux_dockerfile, + gen_windows_dockerfile, +) +from launch.core.runtime import available_platforms + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # +def supported_integration_platforms() -> set[str]: + system = platform.system().lower() + if system == "windows": + return {"windows"} + if system == "linux": + platforms = {"linux", "android"} + #if os.environ.get("REPOLAUNCH_RUN_MACOS_INTEGRATION") == "1" and os.path.exists("/dev/kvm"): + # platforms.add("macos") + return platforms + return set() + + +def parsed_instances(raw_instances: list[str]) -> list[dict[str, Any]]: + """The MOCK_PLATFORM_INSTANCES values are lists of JSON strings.""" + return [json.loads(raw) for raw in raw_instances] + + +def render_dockerfile(runtime_platform: str, instance: dict[str, Any]) -> str: + """Render the dockerfile for one instance, dispatching on platform. + + Note the generators take the ``docker_image_layers`` sub-dict (LayerInfo), not the + whole instance. + """ + layers = instance["docker_image_layers"] + if runtime_platform == "windows": + return gen_windows_dockerfile(layers) + if runtime_platform in ("linux", "android"): + return gen_linux_dockerfile(layers) + raise ValueError(f"Unsupported platform: {runtime_platform}") + + +def all_commands(instance: dict[str, Any]) -> list[str]: + layers = instance["docker_image_layers"] + return list(layers.get("setup_layer") or []) + list(layers.get("organize_layer") or []) + + +def check_docker_existance() -> None: + """Skip (not fail) when the docker CLI / daemon is unavailable on this host.""" + subprocess.run( + ["docker", "ps"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=60, + ) + + +def write_dockerfile(tmp_path: Path, dockerfile: str) -> Path: + path = tmp_path / f"Dockerfile_{uuid.uuid4().hex[:8]}" + path.write_text(dockerfile, encoding="utf-8") + return path + + +def run_docker(args: list[str]) -> subprocess.CompletedProcess: + """Run a docker command with all output going straight to the screen. + + stdout/stderr are inherited (not captured), so the build log streams live. Run pytest + with -s to see it in real time, or -rA to have pytest replay it for passing tests. + """ + print("\n$ " + " ".join(args), flush=True) + return subprocess.run(args, timeout=3600) + + +# Parametrization shared by the integration tests. The second tuple element keeps the +# original ``base_image`` slot name but actually carries the raw instance list for the +# platform (each item is a JSON string). +_PLATFORM_PARAMS = [ + pytest.param("linux", MOCK_PLATFORM_INSTANCES["linux"], id="linux"), + pytest.param("android", MOCK_PLATFORM_INSTANCES["android"], id="android"), + pytest.param("windows", MOCK_PLATFORM_INSTANCES["windows"], id="windows"), + #pytest.param("macos", "sickcodes/docker-osx:auto", id="macos"), +] + + +# --------------------------------------------------------------------------- # +# unit tests (no docker) -- always run; lock the generator's output contract +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize(("runtime_platform", "raw_instances"), _PLATFORM_PARAMS) +def test_dockerfile_contains_base_image_and_all_commands(runtime_platform, raw_instances): + """Every command (and the base image) must be present in the rendered dockerfile.""" + for instance in parsed_instances(raw_instances): + dockerfile = render_dockerfile(runtime_platform, instance) + assert isinstance(dockerfile, str) + assert instance["docker_image_layers"]["base_image"] in dockerfile + + for cmd in all_commands(instance): + if runtime_platform == "windows": + # windows commands are sentinel-encoded; check the encoded form is present. + assert _encode_windows_command(cmd) in dockerfile + else: + # linux commands are embedded verbatim (possibly across several lines). + first_line = cmd.strip().splitlines()[0] + assert first_line in dockerfile + + +@pytest.mark.parametrize(("runtime_platform", "raw_instances"), _PLATFORM_PARAMS) +def test_dockerfile_has_exactly_two_layers(runtime_platform, raw_instances): + # Every instance has a setup and an organize layer, each rendered as exactly one RUN + # instruction (notes 3 & 4). True for both generators: windows emits `RUN