diff --git a/bin/local-dev/README.md b/bin/local-dev/README.md index e1a1f175fcb..bbde2a81d44 100644 --- a/bin/local-dev/README.md +++ b/bin/local-dev/README.md @@ -103,6 +103,26 @@ The script keeps logs, PIDs, build stamps, and animated phase markers under env var). It's safe to `rm -rf` between runs — it'll be recreated on the next invocation. +## Rebuilding the Jupyter image + +`jupyter` is the only managed service that runs from a Texera-built image instead of +natively, so edits to its customizations under +`notebook-migration-service/src/main/resources/` (`custom.js`, `custom.css`, +`start-texera-jupyter.sh`) do nothing until the image is rebuilt. CI publishes it, but a +local edit needs a local build under the same tag: + +```sh +docker build -f bin/dockerfiles/jupyter.dockerfile -t ghcr.io/apache/texera-jupyter:latest . +bin/local-dev.sh up +``` + +Delete that local tag when you are done, otherwise it shadows the published image and you +keep running your old build: + +```sh +docker rmi ghcr.io/apache/texera-jupyter:latest +``` + ## Adding a new managed service 1. Drop the launch command into `main.sh`'s `start_one` switch. diff --git a/bin/local-dev/docker-compose.override.yml b/bin/local-dev/docker-compose.override.yml index 94156e9e56f..3d51fd1cf63 100644 --- a/bin/local-dev/docker-compose.override.yml +++ b/bin/local-dev/docker-compose.override.yml @@ -54,3 +54,12 @@ services: lakekeeper-init: environment: STORAGE_S3_ENDPOINT: ${STORAGE_S3_ENDPOINT:-http://localhost:9000} + + # The default .env points TEXERA_ORIGIN at the nginx origin (:8080), but + # local-dev has no nginx — the Angular dev server serves the app on :4200. + # That origin is what Jupyter puts in its iframe CSP frame-ancestors and + # what custom.js uses as the postMessage targetOrigin, so leaving it at + # :8080 makes the browser refuse to render the notebook panel. + jupyter: + environment: + TEXERA_ORIGIN: ${TEXERA_ORIGIN:-http://localhost:4200} diff --git a/bin/local-dev/main.sh b/bin/local-dev/main.sh index 08e8cd24953..67d3b2fdee5 100755 --- a/bin/local-dev/main.sh +++ b/bin/local-dev/main.sh @@ -104,12 +104,13 @@ # file-service :9092 JVM (sbt FileService) # workflow-compiling-service :9090 JVM (sbt WorkflowCompilingService) # computing-unit-managing-service :8082 JVM (sbt ComputingUnitManagingService) +# notebook-migration-service :9098 JVM (sbt NotebookMigrationService) # texera-web :8080 JVM (sbt WorkflowExecutionService, amber) # computing-unit-master :8085 JVM (rides amber dist; no own sbt project) # agent-service :3001 Bun --watch (cd agent-service && bun run dev) # frontend :4200 ng serve via cd frontend && yarn start # -# Docker infra (postgres / minio / lakefs / lakekeeper / litellm) IS managed +# Docker infra (postgres / minio / lakefs / lakekeeper / litellm / jupyter) IS managed # here: `up` brings it up via `docker compose` (project texera-local-dev) and # `down` tears down any docker targets. The script warns if expected ports are # unreachable. Before any sbt build the postgres schema is reconciled: a fresh @@ -870,12 +871,14 @@ SERVICES=( lakefs lakekeeper litellm + jupyter config-service access-control-service file-service workflow-compiling-service computing-unit-master computing-unit-managing-service + notebook-migration-service texera-web agent-service frontend @@ -893,6 +896,7 @@ amap_set SVC_TYPE minio docker; amap_set SVC_PORT minio 9000; amap_set amap_set SVC_TYPE lakefs docker; amap_set SVC_PORT lakefs 8000; amap_set SVC_CWD lakefs "." amap_set SVC_TYPE lakekeeper docker; amap_set SVC_PORT lakekeeper 8181; amap_set SVC_CWD lakekeeper "." amap_set SVC_TYPE litellm docker; amap_set SVC_PORT litellm 4000; amap_set SVC_CWD litellm "." +amap_set SVC_TYPE jupyter docker; amap_set SVC_PORT jupyter 9100; amap_set SVC_CWD jupyter "." amap_set SVC_TYPE config-service jvm amap_set SVC_PORT config-service 9094 @@ -930,6 +934,15 @@ amap_set SVC_ZIP_GLOB workflow-compiling-service "workflow-compiling-service/t amap_set SVC_UNZIP_DEST workflow-compiling-service "target/" amap_set SVC_HEALTH workflow-compiling-service "/api/healthcheck" +amap_set SVC_TYPE notebook-migration-service jvm +amap_set SVC_PORT notebook-migration-service 9098 +amap_set SVC_SBT notebook-migration-service NotebookMigrationService +amap_set SVC_LAUNCHER notebook-migration-service "target/notebook-migration-service-${TEXERA_VERSION}/bin/notebook-migration-service" +amap_set SVC_CWD notebook-migration-service "." +amap_set SVC_ZIP_GLOB notebook-migration-service "notebook-migration-service/target/universal/notebook-migration-service-*.zip" +amap_set SVC_UNZIP_DEST notebook-migration-service "target/" +amap_set SVC_HEALTH notebook-migration-service "/api/healthcheck" + amap_set SVC_TYPE computing-unit-managing-service jvm amap_set SVC_PORT computing-unit-managing-service 8082 amap_set SVC_SBT computing-unit-managing-service ComputingUnitManagingService @@ -983,8 +996,8 @@ DOCKER_PROJECT="texera-local-dev" DOCKER_COMPOSE_FILE="$SELF_ROOT/bin/single-node/docker-compose.yml" DOCKER_OVERLAY_FILE="$SELF_ROOT/bin/local-dev/docker-compose.override.yml" DOCKER_ENV_FILE="$SELF_ROOT/bin/single-node/.env" -DOCKER_INFRA_SERVICES=(postgres minio minio-init lakefs lakekeeper-migrate lakekeeper lakekeeper-init litellm) -DOCKER_INFRA_LONGLIVED=(postgres minio lakefs lakekeeper litellm) # exclude one-shot init jobs +DOCKER_INFRA_SERVICES=(postgres minio minio-init lakefs lakekeeper-migrate lakekeeper lakekeeper-init litellm jupyter) +DOCKER_INFRA_LONGLIVED=(postgres minio lakefs lakekeeper litellm jupyter) # exclude one-shot init jobs # Build the array of -f flags: base single-node compose + local-dev overlay # (the overlay publishes infra ports to the host, which the upstream compose @@ -1679,7 +1692,7 @@ infra_up() { # --progress=tty forces it even if stdout looks like a pipe. docker compose --progress auto -p "$DOCKER_PROJECT" --env-file "$DOCKER_ENV_FILE" "${files[@]}" \ up -d "${DOCKER_INFRA_SERVICES[@]}" - tui_ok "infra: 5 containers up" + tui_ok "infra: ${#DOCKER_INFRA_LONGLIVED[@]} containers up" } infra_down() { diff --git a/bin/local-dev/tui.py b/bin/local-dev/tui.py index e3892cc030e..8a5b7a54190 100644 --- a/bin/local-dev/tui.py +++ b/bin/local-dev/tui.py @@ -283,6 +283,7 @@ def _jvm(name: str, port: int, project: Optional[str], own_src: str) -> Service: Service("lakefs", "docker", 8000), Service("lakekeeper", "docker", 8181), Service("litellm", "docker", 4000), + Service("jupyter", "docker", 9100), _jvm("config-service", 9094, "ConfigService", "config-service/src"), _jvm("access-control-service", 9096, "AccessControlService", @@ -295,6 +296,8 @@ def _jvm(name: str, port: int, project: Optional[str], own_src: str) -> Service: "amber/src"), _jvm("computing-unit-managing-service", 8082, "ComputingUnitManagingService", "computing-unit-managing-service/src"), + _jvm("notebook-migration-service", 9098, "NotebookMigrationService", + "notebook-migration-service/src"), _jvm("texera-web", 8080, "WorkflowExecutionService", "amber/src"), Service("agent-service", "bun", 3001),