chore: migrate to flyte-sandbox-bundled and fix CI (#324)

honnix · claude · web-flow · commit 0b38d5177916 · 2026-03-24T15:50:19.000+01:00
Co-authored-by: Claude Opus 4.6 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/flytekit-java/src/main/java/org/flyte/flytekit/SdkConfig.java b/flytekit-java/src/main/java/org/flyte/flytekit/SdkConfig.java
@@ -27,9 +27,9 @@
 abstract class SdkConfig {
 
   // VisibleForTesting
-  static final String DOMAIN_ENV_VAR = "FLYTE_INTERNAL_DOMAIN";
-  static final String PROJECT_ENV_VAR = "FLYTE_INTERNAL_PROJECT";
-  static final String VERSION_ENV_VAR = "FLYTE_INTERNAL_VERSION";
+  static final String DOMAIN_ENV_VAR = "FLYTE_INTERNAL_TASK_DOMAIN";
+  static final String PROJECT_ENV_VAR = "FLYTE_INTERNAL_TASK_PROJECT";
+  static final String VERSION_ENV_VAR = "FLYTE_INTERNAL_TASK_VERSION";
 
   public abstract String project();
 
diff --git a/flytekit-local-engine/src/test/java/org/flyte/localengine/LocalEngineTest.java b/flytekit-local-engine/src/test/java/org/flyte/localengine/LocalEngineTest.java
@@ -501,9 +501,9 @@ public static Stream<Arguments> testBranchNodesProvider() {
   private static Map<String, WorkflowTemplate> loadWorkflows() {
     Map<String, String> env =
         ImmutableMap.of(
-            "FLYTE_INTERNAL_DOMAIN", "development",
-            "FLYTE_INTERNAL_VERSION", "test",
-            "FLYTE_INTERNAL_PROJECT", "flytetester");
+            "FLYTE_INTERNAL_TASK_DOMAIN", "development",
+            "FLYTE_INTERNAL_TASK_VERSION", "test",
+            "FLYTE_INTERNAL_TASK_PROJECT", "flytetester");
 
     Map<WorkflowIdentifier, WorkflowTemplate> registrarWorkflows =
         loadAll(WorkflowTemplateRegistrar.class, env);
@@ -515,9 +515,9 @@ private static Map<String, WorkflowTemplate> loadWorkflows() {
   private static Map<String, RunnableTask> loadTasks() {
     Map<String, String> env =
         ImmutableMap.of(
-            "FLYTE_INTERNAL_DOMAIN", "development",
-            "FLYTE_INTERNAL_VERSION", "test",
-            "FLYTE_INTERNAL_PROJECT", "flytetester");
+            "FLYTE_INTERNAL_TASK_DOMAIN", "development",
+            "FLYTE_INTERNAL_TASK_VERSION", "test",
+            "FLYTE_INTERNAL_TASK_PROJECT", "flytetester");
 
     Map<TaskIdentifier, RunnableTask> registrarRunnableTasks =
         loadAll(RunnableTaskRegistrar.class, env);
diff --git a/integration-tests/src/test/java/org/flyte/SerializeJavaIT.java b/integration-tests/src/test/java/org/flyte/SerializeJavaIT.java
@@ -36,14 +36,15 @@ class SerializeJavaIT extends Fixtures {
   @Test
   void testSerializeWorkflows() {
     try {
-      File current = new File("target/protos");
-      File tempDir = managed.resolve(current.getAbsolutePath()).toFile();
-      boolean created = tempDir.mkdir();
-      if (!created) {
+      // Path must be relative to project root since jflyte runs in a container
+      // with the project root as working directory
+      String serializePath = "integration-tests/target/protos";
+      File tempDir = new File("../" + serializePath);
+      if (!tempDir.mkdirs() && !tempDir.exists()) {
         throw new IOException("Unable to create path");
       }
 
-      CLIENT.serializeWorkflows(CLASSPATH, tempDir.getPath());
+      CLIENT.serializeWorkflows(CLASSPATH, serializePath);
 
       boolean hasFibonacciWorkflow =
           Stream.of(tempDir.list())
diff --git a/integration-tests/src/test/java/org/flyte/utils/FlyteSandboxClient.java b/integration-tests/src/test/java/org/flyte/utils/FlyteSandboxClient.java
@@ -47,7 +47,7 @@ public static FlyteSandboxClient create() {
     String version = String.valueOf(System.currentTimeMillis());
 
     String address = FlyteSandboxContainer.INSTANCE.getHost();
-    int port = FlyteSandboxContainer.INSTANCE.getMappedPort(30081);
+    int port = FlyteSandboxContainer.INSTANCE.getMappedPort(30080);
 
     ManagedChannel channel =
         ManagedChannelBuilder.forTarget(address + ":" + port).usePlaintext().enableRetry().build();
diff --git a/integration-tests/src/test/java/org/flyte/utils/FlyteSandboxContainer.java b/integration-tests/src/test/java/org/flyte/utils/FlyteSandboxContainer.java
@@ -25,6 +25,7 @@
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.time.Duration;
+import java.util.Map;
 import org.apache.commons.compress.utils.IOUtils;
 import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.BindMode;
@@ -33,7 +34,8 @@
 
 public class FlyteSandboxContainer extends GenericContainer<FlyteSandboxContainer> {
 
-  public static final String IMAGE_NAME = "ghcr.io/flyteorg/flyte-sandbox:v1.9.1";
+  public static final String IMAGE_NAME =
+      "ghcr.io/flyteorg/flyte-sandbox-bundled:sha-f3ab1b7480bad4072f7ecb695660fdf47032a6c4";
 
   public static final FlyteSandboxContainer INSTANCE =
       new FlyteSandboxContainer()
@@ -48,13 +50,14 @@ public class FlyteSandboxContainer extends GenericContainer<FlyteSandboxContaine
   private static void startContainer() {
     INSTANCE.start();
 
-    // Flyte sandbox uses Docker in Docker, we have to copy jflyte container into inner Docker
+    // Flyte sandbox-bundled uses k3s with containerd (not Docker-in-Docker),
+    // so we use ctr to import the jflyte image into the inner containerd
     // otherwise, flytepropeller can't use the right version for pod execution
 
     DockerClient client = DockerClientFactory.instance().client();
     try (InputStream imageInputStream = client.saveImageCmd(JFlyteContainer.IMAGE_NAME).exec()) {
 
-      try (OutputStream outputStream = Files.newOutputStream(Paths.get("target/jflyte.tar.gz"))) {
+      try (OutputStream outputStream = Files.newOutputStream(Paths.get("target/jflyte.tar"))) {
         IOUtils.copy(imageInputStream, outputStream);
       }
 
@@ -64,7 +67,14 @@ private static void startContainer() {
 
       ExecResult execResult =
           INSTANCE.execInContainer(
-              "docker", "load", "-i", "integration-tests/target/jflyte.tar.gz");
+              "ctr",
+              "--address",
+              "/run/k3s/containerd/containerd.sock",
+              "--namespace",
+              "k8s.io",
+              "images",
+              "import",
+              "integration-tests/target/jflyte.tar");
 
       if (execResult.getExitCode() != 0) {
         throw new RuntimeException(execResult.getStderr() + " " + execResult.getStdout());
@@ -75,6 +85,68 @@ private static void startContainer() {
       Thread.currentThread().interrupt();
       throw new RuntimeException("failed to load jflyte image", e);
     }
+
+    // The sandbox manifest has FLYTE_PLATFORM_INSECURE as a YAML boolean (true)
+    // which may not be injected correctly as a string env var. Patch the configmap
+    // to use a quoted string value and restart the flyte-sandbox pod.
+    patchInsecureEnvVar();
+  }
+
+  private static void patchInsecureEnvVar() {
+    try {
+      ExecResult patchResult =
+          INSTANCE.execInContainer(
+              "kubectl",
+              "get",
+              "configmap",
+              "-n",
+              "flyte",
+              "flyte-sandbox-config",
+              "-o",
+              "jsonpath={.data.100-inline-config\\.yaml}");
+
+      if (patchResult.getExitCode() != 0) {
+        throw new RuntimeException("Failed to read configmap: " + patchResult.getStderr());
+      }
+
+      String config = patchResult.getStdout();
+      if (config.contains("FLYTE_PLATFORM_INSECURE: true")) {
+        String patched =
+            config.replace("FLYTE_PLATFORM_INSECURE: true", "FLYTE_PLATFORM_INSECURE: 'true'");
+
+        ExecResult applyResult =
+            INSTANCE.execInContainer(
+                "sh",
+                "-c",
+                "kubectl create configmap flyte-sandbox-config -n flyte"
+                    + " --from-literal='100-inline-config.yaml="
+                    + patched.replace("'", "'\"'\"'")
+                    + "' --dry-run=client -o yaml | kubectl apply -f -");
+
+        if (applyResult.getExitCode() != 0) {
+          throw new RuntimeException("Failed to patch configmap: " + applyResult.getStderr());
+        }
+
+        // Restart flyte-sandbox pod to pick up the new config
+        INSTANCE.execInContainer(
+            "kubectl", "rollout", "restart", "deployment/flyte-sandbox", "-n", "flyte");
+
+        // Wait for the rollout to complete
+        INSTANCE.execInContainer(
+            "kubectl",
+            "rollout",
+            "status",
+            "deployment/flyte-sandbox",
+            "-n",
+            "flyte",
+            "--timeout=120s");
+      }
+    } catch (IOException e) {
+      throw new UncheckedIOException("failed to patch insecure env var", e);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new RuntimeException("failed to patch insecure env var", e);
+    }
   }
 
   FlyteSandboxContainer() {
@@ -84,23 +156,31 @@ private static void startContainer() {
 
     withPrivilegedMode(true);
 
+    // k3s requires tmpfs mounts and cgroupns=host on Linux (CI)
+    withTmpFs(Map.of("/run", "", "/var/run", ""));
+    withCreateContainerCmdModifier(
+        cmd ->
+            cmd.getHostConfig()
+                .withCgroupnsMode("host")
+                // host.docker.internal is needed by the bootstrap to template manifests;
+                // Docker Desktop adds it automatically, but Linux Docker does not
+                .withExtraHosts("host.docker.internal:host-gateway"));
+
     withNetworkAliases("flyte");
 
     withWorkingDirectory(workingDir);
     withFileSystemBind(workingDir, workingDir, BindMode.READ_ONLY);
 
     withExposedPorts(
-        30081, // flyteadmin
-        30082, // k8s dashboard
-        30084, // minio
-        30086 // k8s api
+        30080, // envoy proxy (flyteadmin http + grpc)
+        30002 // minio
         );
 
     withReuse(true);
 
     withNetwork(FlyteSandboxNetwork.INSTANCE);
 
-    waitingFor(Wait.forLogMessage(".*Flyte is ready!.*", 1));
+    waitingFor(Wait.forHttp("/healthcheck").forPort(30080));
     withStartupTimeout(Duration.ofMinutes(5));
   }
 
@@ -110,10 +190,8 @@ public void start() {
 
     logger().info("Flyte is ready!");
 
-    String consoleUri = String.format("http://%s:%d/console", getHost(), getMappedPort(30081));
-    String k8sUri = String.format("http://%s:%d", getHost(), getMappedPort(30082));
+    String consoleUri = String.format("http://%s:%d/console", getHost(), getMappedPort(30080));
 
     logger().info("Flyte UI is available at " + consoleUri);
-    logger().info("K8s dashboard is available at " + k8sUri);
   }
 }
diff --git a/integration-tests/src/test/java/org/flyte/utils/JFlyteContainer.java b/integration-tests/src/test/java/org/flyte/utils/JFlyteContainer.java
@@ -30,9 +30,9 @@ class JFlyteContainer extends GenericContainer<JFlyteContainer> {
   static final String IMAGE_NAME;
   static final Map<String, String> envVars =
       ImmutableMap.<String, String>builder()
-          .put("FLYTE_PLATFORM_URL", "flyte:30081")
+          .put("FLYTE_PLATFORM_URL", "flyte:30080")
           .put("FLYTE_PLATFORM_INSECURE", "True")
-          .put("FLYTE_AWS_ENDPOINT", "http://flyte:30084")
+          .put("FLYTE_AWS_ENDPOINT", "http://flyte:30002")
           .put("FLYTE_AWS_ACCESS_KEY_ID", "minio")
           .put("FLYTE_AWS_SECRET_ACCESS_KEY", "miniostorage")
           .put("FLYTE_STAGING_LOCATION", "s3://my-s3-bucket")
diff --git a/integration-tests/src/test/resources/docker-java.properties b/integration-tests/src/test/resources/docker-java.properties
@@ -0,0 +1 @@
+api.version=1.44
diff --git a/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ExecutionConfig.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ExecutionConfig.java
@@ -33,9 +33,9 @@ public abstract class ExecutionConfig {
 
   public static ExecutionConfig load() {
     return ExecutionConfig.builder()
-        .project(getenv("FLYTE_INTERNAL_PROJECT"))
-        .domain(getenv("FLYTE_INTERNAL_DOMAIN"))
-        .version(getenv("FLYTE_INTERNAL_VERSION"))
+        .project(getenv("FLYTE_INTERNAL_TASK_PROJECT"))
+        .domain(getenv("FLYTE_INTERNAL_TASK_DOMAIN"))
+        .version(getenv("FLYTE_INTERNAL_TASK_VERSION"))
         .image(getenv("FLYTE_INTERNAL_IMAGE"))
         .build();
   }
diff --git a/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProjectClosure.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProjectClosure.java
@@ -212,9 +212,9 @@ static ProjectClosure load(
             .put("JFLYTE_DOMAIN", config.domain())
             .put("JFLYTE_PROJECT", config.project())
             .put("JFLYTE_VERSION", config.version())
-            .put("FLYTE_INTERNAL_DOMAIN", config.domain())
-            .put("FLYTE_INTERNAL_PROJECT", config.project())
-            .put("FLYTE_INTERNAL_VERSION", config.version())
+            .put("FLYTE_INTERNAL_TASK_DOMAIN", config.domain())
+            .put("FLYTE_INTERNAL_TASK_PROJECT", config.project())
+            .put("FLYTE_INTERNAL_TASK_VERSION", config.version())
             .build();
 
     // 1. load classes, and create templates
diff --git a/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocal.java b/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocal.java
@@ -64,9 +64,9 @@ public Integer call() {
 
     Map<String, String> env =
         ImmutableMap.of(
-            "FLYTE_INTERNAL_DOMAIN", "development",
-            "FLYTE_INTERNAL_VERSION", "test",
-            "FLYTE_INTERNAL_PROJECT", "flytetester");
+            "FLYTE_INTERNAL_TASK_DOMAIN", "development",
+            "FLYTE_INTERNAL_TASK_VERSION", "test",
+            "FLYTE_INTERNAL_TASK_PROJECT", "flytetester");
 
     Map<String, RunnableTask> runnableTasks = ExecuteLocalLoader.loadTasks(modules, env);
     Map<String, DynamicWorkflowTask> dynamicWorkflowTasks =
diff --git a/pom.xml b/pom.xml
@@ -379,7 +379,7 @@
       <dependency>
         <groupId>net.java.dev.jna</groupId>
         <artifactId>jna</artifactId>
-        <version>5.12.1</version>
+        <version>5.13.0</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson</groupId>
@@ -405,7 +405,7 @@
       <dependency>
         <groupId>org.testcontainers</groupId>
         <artifactId>testcontainers-bom</artifactId>
-        <version>1.19.1</version>
+        <version>1.21.4</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>
diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml
@@ -64,5 +64,11 @@
     <Bug pattern="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"/>
   </Match>
 
+  <!-- HostConfig is always set by Testcontainers before the modifier runs -->
+  <Match>
+    <Class name="org.flyte.utils.FlyteSandboxContainer"/>
+    <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
+  </Match>
+
 </FindBugsFilter>
 
