feat(notebook-migration, local-dev): manage the notebook migration service and JupyterLab

bin/local-dev/README.md

@@ -103,6 +103,26 @@ The script keeps logs, PIDs, build stamps, and animated phase markers under
 env var). It's safe to `rm -rf` between runs — it'll be recreated on the next
 invocation.
 
+## Rebuilding the Jupyter image
+
+`jupyter` is the only managed service that runs from a Texera-built image instead of
+natively, so edits to its customizations under
+`notebook-migration-service/src/main/resources/` (`custom.js`, `custom.css`,
+`start-texera-jupyter.sh`) do nothing until the image is rebuilt. CI publishes it, but a
+local edit needs a local build under the same tag:
+
+```sh
+docker build -f bin/dockerfiles/jupyter.dockerfile -t ghcr.io/apache/texera-jupyter:latest .
+bin/local-dev.sh up
+```
+
+Delete that local tag when you are done, otherwise it shadows the published image and you
+keep running your old build:
+
+```sh
+docker rmi ghcr.io/apache/texera-jupyter:latest
+```
+
 ## Adding a new managed service
 
 1. Drop the launch command into `main.sh`'s `start_one` switch.

bin/local-dev/docker-compose.override.yml

@@ -54,3 +54,12 @@ services:
   lakekeeper-init:
     environment:
       STORAGE_S3_ENDPOINT: ${STORAGE_S3_ENDPOINT:-http://localhost:9000}
+
+  # The default .env points TEXERA_ORIGIN at the nginx origin (:8080), but
+  # local-dev has no nginx — the Angular dev server serves the app on :4200.
+  # That origin is what Jupyter puts in its iframe CSP frame-ancestors and
+  # what custom.js uses as the postMessage targetOrigin, so leaving it at
+  # :8080 makes the browser refuse to render the notebook panel.
+  jupyter:
+    environment:
+      TEXERA_ORIGIN: ${TEXERA_ORIGIN:-http://localhost:4200}

bin/local-dev/main.sh

@@ -104,12 +104,13 @@
 #   file-service                   :9092  JVM (sbt FileService)
 #   workflow-compiling-service     :9090  JVM (sbt WorkflowCompilingService)
 #   computing-unit-managing-service :8082 JVM (sbt ComputingUnitManagingService)
+#   notebook-migration-service     :9098  JVM (sbt NotebookMigrationService)
 #   texera-web                     :8080  JVM (sbt WorkflowExecutionService, amber)
 #   computing-unit-master          :8085  JVM (rides amber dist; no own sbt project)
 #   agent-service                  :3001  Bun --watch (cd agent-service && bun run dev)
 #   frontend                       :4200  ng serve via cd frontend && yarn start
 #
-# Docker infra (postgres / minio / lakefs / lakekeeper / litellm) IS managed
+# Docker infra (postgres / minio / lakefs / lakekeeper / litellm / jupyter) IS managed
 # here: `up` brings it up via `docker compose` (project texera-local-dev) and
 # `down` tears down any docker targets. The script warns if expected ports are
 # unreachable. Before any sbt build the postgres schema is reconciled: a fresh
@@ -870,12 +871,14 @@ SERVICES=(
     lakefs
     lakekeeper
     litellm
+    jupyter
     config-service
     access-control-service
     file-service
     workflow-compiling-service
     computing-unit-master
     computing-unit-managing-service
+    notebook-migration-service
     texera-web
     agent-service
     frontend
@@ -893,6 +896,7 @@ amap_set SVC_TYPE minio      docker; amap_set SVC_PORT minio      9000; amap_set
 amap_set SVC_TYPE lakefs     docker; amap_set SVC_PORT lakefs     8000; amap_set SVC_CWD lakefs     "."
 amap_set SVC_TYPE lakekeeper docker; amap_set SVC_PORT lakekeeper 8181; amap_set SVC_CWD lakekeeper "."
 amap_set SVC_TYPE litellm    docker; amap_set SVC_PORT litellm    4000; amap_set SVC_CWD litellm    "."
+amap_set SVC_TYPE jupyter    docker; amap_set SVC_PORT jupyter    9100; amap_set SVC_CWD jupyter    "."
 
 amap_set SVC_TYPE       config-service jvm
 amap_set SVC_PORT       config-service 9094
@@ -930,6 +934,15 @@ amap_set SVC_ZIP_GLOB   workflow-compiling-service "workflow-compiling-service/t
 amap_set SVC_UNZIP_DEST workflow-compiling-service "target/"
 amap_set SVC_HEALTH     workflow-compiling-service "/api/healthcheck"
 
+amap_set SVC_TYPE       notebook-migration-service jvm
+amap_set SVC_PORT       notebook-migration-service 9098
+amap_set SVC_SBT        notebook-migration-service NotebookMigrationService
+amap_set SVC_LAUNCHER   notebook-migration-service "target/notebook-migration-service-${TEXERA_VERSION}/bin/notebook-migration-service"
+amap_set SVC_CWD        notebook-migration-service "."
+amap_set SVC_ZIP_GLOB   notebook-migration-service "notebook-migration-service/target/universal/notebook-migration-service-*.zip"
+amap_set SVC_UNZIP_DEST notebook-migration-service "target/"
+amap_set SVC_HEALTH     notebook-migration-service "/api/healthcheck"
+
 amap_set SVC_TYPE       computing-unit-managing-service jvm
 amap_set SVC_PORT       computing-unit-managing-service 8082
 amap_set SVC_SBT        computing-unit-managing-service ComputingUnitManagingService
@@ -983,8 +996,8 @@ DOCKER_PROJECT="texera-local-dev"
 DOCKER_COMPOSE_FILE="$SELF_ROOT/bin/single-node/docker-compose.yml"
 DOCKER_OVERLAY_FILE="$SELF_ROOT/bin/local-dev/docker-compose.override.yml"
 DOCKER_ENV_FILE="$SELF_ROOT/bin/single-node/.env"
-DOCKER_INFRA_SERVICES=(postgres minio minio-init lakefs lakekeeper-migrate lakekeeper lakekeeper-init litellm)
-DOCKER_INFRA_LONGLIVED=(postgres minio lakefs lakekeeper litellm)  # exclude one-shot init jobs
+DOCKER_INFRA_SERVICES=(postgres minio minio-init lakefs lakekeeper-migrate lakekeeper lakekeeper-init litellm jupyter)
+DOCKER_INFRA_LONGLIVED=(postgres minio lakefs lakekeeper litellm jupyter)  # exclude one-shot init jobs
 
 # Build the array of -f flags: base single-node compose + local-dev overlay
 # (the overlay publishes infra ports to the host, which the upstream compose
@@ -1679,7 +1692,7 @@ infra_up() {
     # --progress=tty forces it even if stdout looks like a pipe.
     docker compose --progress auto -p "$DOCKER_PROJECT" --env-file "$DOCKER_ENV_FILE" "${files[@]}" \
         up -d "${DOCKER_INFRA_SERVICES[@]}"
-    tui_ok "infra: 5 containers up"
+    tui_ok "infra: ${#DOCKER_INFRA_LONGLIVED[@]} containers up"
 }
 
 infra_down() {

bin/local-dev/tui.py

@@ -283,6 +283,7 @@ def _jvm(name: str, port: int, project: Optional[str], own_src: str) -> Service:
     Service("lakefs",     "docker", 8000),
     Service("lakekeeper", "docker", 8181),
     Service("litellm",    "docker", 4000),
+    Service("jupyter",    "docker", 9100),
     _jvm("config-service",                  9094, "ConfigService",
          "config-service/src"),
     _jvm("access-control-service",          9096, "AccessControlService",
@@ -295,6 +296,8 @@ def _jvm(name: str, port: int, project: Optional[str], own_src: str) -> Service:
          "amber/src"),
     _jvm("computing-unit-managing-service", 8082, "ComputingUnitManagingService",
          "computing-unit-managing-service/src"),
+    _jvm("notebook-migration-service",      9098, "NotebookMigrationService",
+         "notebook-migration-service/src"),
     _jvm("texera-web",                      8080, "WorkflowExecutionService",
          "amber/src"),
     Service("agent-service", "bun",  3001),