feat(notebook-migration): resolve the Jupyter URL and token per user

common/config/src/main/resources/kubernetes.conf

@@ -54,4 +54,33 @@ kubernetes {
   # GPU resource key used in Kubernetes (vendor-specific)
   computing-unit-gpu-resource-key = "nvidia.com/gpu"
   computing-unit-gpu-resource-key = ${?KUBERNETES_COMPUTING_UNIT_GPU_RESOURCE_KEY}
+
+  # Per-user JupyterLab pods. Separate from `enabled` above, so a deployment can run
+  # computing units on Kubernetes without per-user Jupyter. While this is off, the
+  # notebook migration service uses the single Jupyter from storage.jupyter.
+  jupyter-enabled = false
+  jupyter-enabled = ${?KUBERNETES_JUPYTER_ENABLED}
+
+  jupyter-namespace = "texera-jupyter-pool"
+  jupyter-namespace = ${?KUBERNETES_JUPYTER_NAMESPACE}
+
+  jupyter-service-name = "jupyter-svc"
+  jupyter-service-name = ${?KUBERNETES_JUPYTER_SERVICE_NAME}
+
+  jupyter-image-name = "ghcr.io/apache/texera-jupyter:latest"
+  jupyter-image-name = ${?KUBERNETES_JUPYTER_IMAGE_NAME}
+
+  jupyter-port-num = 8888
+
+  jupyter-cpu-limit = "1"
+  jupyter-cpu-limit = ${?KUBERNETES_JUPYTER_CPU_LIMIT}
+
+  jupyter-memory-limit = "2Gi"
+  jupyter-memory-limit = ${?KUBERNETES_JUPYTER_MEMORY_LIMIT}
+
+  # Browser-facing address, with {uid} substituted. The in-network pod name does not
+  # resolve from the browser, so a deployment that publishes Jupyter sets this; empty
+  # falls back to the in-network address.
+  jupyter-public-url-template = ""
+  jupyter-public-url-template = ${?KUBERNETES_JUPYTER_PUBLIC_URL_TEMPLATE}
 }

common/config/src/main/resources/storage.conf

@@ -177,7 +177,8 @@ storage {
         password = ${?STORAGE_JDBC_PASSWORD}
     }
 
-    # Configurations of the JupyterLab service
+    # The single JupyterLab used when per-user provisioning (kubernetes.jupyter-enabled)
+    # is off, which is how the single-node and local-dev deployments run.
     jupyter {
         internal-url = "http://localhost:9100"
         internal-url = ${?STORAGE_JUPYTER_INTERNAL_URL}
@@ -188,5 +189,11 @@ storage {
         # Read from the same JUPYTER_TOKEN env var as the Jupyter container
         token = "texera"
         token = ${?JUPYTER_TOKEN}
+
+        # HMAC key each per-user Jupyter token is derived from, so no token is stored.
+        # Required when per-user Jupyter is on, and must stay stable across restarts and
+        # replicas or previously issued tokens stop matching.
+        token-secret = ""
+        token-secret = ${?JUPYTER_TOKEN_SECRET}
     }
 }

common/config/src/main/scala/org/apache/texera/common/config/KubernetesConfig.scala

@@ -64,4 +64,17 @@ object KubernetesConfig {
 
   // GPU resource key used directly in Kubernetes resource specifications
   val gpuResourceKey: String = conf.getString("kubernetes.computing-unit-gpu-resource-key")
+
+  // Per-user JupyterLab pods, gated independently of computing units.
+  val jupyterEnabled: Boolean = conf.getBoolean("kubernetes.jupyter-enabled")
+  val jupyterNamespace: String = conf.getString("kubernetes.jupyter-namespace")
+  val jupyterServiceName: String = conf.getString("kubernetes.jupyter-service-name")
+  val jupyterImageName: String = conf.getString("kubernetes.jupyter-image-name")
+  val jupyterPortNumber: Int = conf.getInt("kubernetes.jupyter-port-num")
+  val jupyterCpuLimit: String = conf.getString("kubernetes.jupyter-cpu-limit")
+  val jupyterMemoryLimit: String = conf.getString("kubernetes.jupyter-memory-limit")
+
+  // Browser-facing address with {uid} substituted; empty means use the in-network one.
+  val jupyterPublicUrlTemplate: String =
+    conf.getString("kubernetes.jupyter-public-url-template")
 }

common/config/src/main/scala/org/apache/texera/common/config/StorageConfig.scala

@@ -160,4 +160,7 @@ object StorageConfig {
   val jupyterInternalURL: String = conf.getString("storage.jupyter.internal-url")
   val jupyterPublicURL: String = conf.getString("storage.jupyter.public-url")
   val jupyterToken: String = conf.getString("storage.jupyter.token")
+
+  // HMAC key for per-user token derivation; empty unless a deployment sets it.
+  val jupyterTokenSecret: String = conf.getString("storage.jupyter.token-secret")
 }

common/config/src/test/scala/org/apache/texera/common/config/KubernetesConfigSpec.scala

@@ -24,7 +24,7 @@ import org.scalatest.matchers.should.Matchers
 
 /**
   * Spec for [[KubernetesConfig]]. Reading each value forces resolution from kubernetes.conf, so a
-  * renamed or mistyped key surfaces here as a ConfigException. Every value except the port number
+  * renamed or mistyped key surfaces here as a ConfigException. Every value except the port numbers
   * carries a `${?ENV}` override, so exact-value assertions are guarded on the env var being unset.
   */
 class KubernetesConfigSpec extends AnyFlatSpec with Matchers {
@@ -70,6 +70,31 @@ class KubernetesConfigSpec extends AnyFlatSpec with Matchers {
     KubernetesConfig.maxNumOfRunningComputingUnitsPerUser should be >= 0
   }
 
+  "KubernetesConfig jupyter settings" should "resolve to their kubernetes.conf defaults" in {
+    KubernetesConfig.jupyterPortNumber shouldBe 8888
+    // Off by default and keyed separately from kubernetes.enabled, so enabling computing
+    // units on Kubernetes never silently enables per-user Jupyter.
+    ifUnset("KUBERNETES_JUPYTER_ENABLED")(KubernetesConfig.jupyterEnabled shouldBe false)
+    ifUnset("KUBERNETES_JUPYTER_NAMESPACE")(
+      KubernetesConfig.jupyterNamespace shouldBe "texera-jupyter-pool"
+    )
+    ifUnset("KUBERNETES_JUPYTER_SERVICE_NAME")(
+      KubernetesConfig.jupyterServiceName shouldBe "jupyter-svc"
+    )
+    ifUnset("KUBERNETES_JUPYTER_IMAGE_NAME")(
+      KubernetesConfig.jupyterImageName shouldBe "ghcr.io/apache/texera-jupyter:latest"
+    )
+    ifUnset("KUBERNETES_JUPYTER_CPU_LIMIT")(KubernetesConfig.jupyterCpuLimit shouldBe "1")
+    ifUnset("KUBERNETES_JUPYTER_MEMORY_LIMIT")(
+      KubernetesConfig.jupyterMemoryLimit shouldBe "2Gi"
+    )
+    // Empty means the browser is handed the in-network address; a deployment that
+    // publishes Jupyter overrides it.
+    ifUnset("KUBERNETES_JUPYTER_PUBLIC_URL_TEMPLATE")(
+      KubernetesConfig.jupyterPublicUrlTemplate shouldBe ""
+    )
+  }
+
   "KubernetesConfig limit options" should "parse into trimmed, non-empty lists" in {
     ifUnset("KUBERNETES_COMPUTING_UNIT_CPU_LIMIT_OPTIONS")(
       KubernetesConfig.cpuLimitOptions shouldBe List("1", "2", "4")

common/config/src/test/scala/org/apache/texera/common/config/StorageConfigSpec.scala

@@ -73,4 +73,12 @@ class StorageConfigSpec extends AnyFlatSpec with Matchers {
       StorageConfig.jupyterPublicURL shouldBe StorageConfig.jupyterInternalURL
     }
   }
+
+  it should "default the token secret to empty so a deployment must set it deliberately" in {
+    // Per-user tokens are derived from this key, so it has no safe default: an empty
+    // value must be caught at start-up rather than silently deriving from nothing.
+    if (sys.env.get("JUPYTER_TOKEN_SECRET").isEmpty) {
+      StorageConfig.jupyterTokenSecret shouldBe ""
+    }
+  }
 }

notebook-migration-service/LICENSE-binary

@@ -224,10 +224,10 @@ Scala/Java jars:
   - com.fasterxml.jackson.core.jackson-annotations-2.18.8.jar
   - com.fasterxml.jackson.core.jackson-core-2.18.8.jar
   - com.fasterxml.jackson.core.jackson-databind-2.18.8.jar
-  - com.fasterxml.jackson.dataformat.jackson-dataformat-yaml-2.16.1.jar
+  - com.fasterxml.jackson.dataformat.jackson-dataformat-yaml-2.17.0.jar
   - com.fasterxml.jackson.datatype.jackson-datatype-guava-2.16.1.jar
   - com.fasterxml.jackson.datatype.jackson-datatype-jdk8-2.16.1.jar
-  - com.fasterxml.jackson.datatype.jackson-datatype-jsr310-2.16.1.jar
+  - com.fasterxml.jackson.datatype.jackson-datatype-jsr310-2.17.0.jar
   - com.fasterxml.jackson.jakarta.rs.jackson-jakarta-rs-base-2.16.1.jar
   - com.fasterxml.jackson.jakarta.rs.jackson-jakarta-rs-json-provider-2.16.1.jar
   - com.fasterxml.jackson.module.jackson-module-blackbird-2.16.1.jar
@@ -242,6 +242,9 @@ Scala/Java jars:
   - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
   - com.google.j2objc.j2objc-annotations-2.8.jar
   - com.helger.profiler-1.1.1.jar
+  - com.squareup.okhttp3.logging-interceptor-3.12.12.jar
+  - com.squareup.okhttp3.okhttp-3.12.12.jar
+  - com.squareup.okio.okio-1.15.0.jar
   - com.thesamet.scalapb.lenses_2.13-0.11.20.jar
   - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar
   - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar
@@ -274,6 +277,32 @@ Scala/Java jars:
   - io.dropwizard.metrics.metrics-json-4.2.25.jar
   - io.dropwizard.metrics.metrics-jvm-4.2.25.jar
   - io.dropwizard.metrics.metrics-logback-4.2.25.jar
+  - io.fabric8.kubernetes-client-6.12.1.jar
+  - io.fabric8.kubernetes-client-api-6.12.1.jar
+  - io.fabric8.kubernetes-httpclient-okhttp-6.12.1.jar
+  - io.fabric8.kubernetes-model-admissionregistration-6.12.1.jar
+  - io.fabric8.kubernetes-model-apiextensions-6.12.1.jar
+  - io.fabric8.kubernetes-model-apps-6.12.1.jar
+  - io.fabric8.kubernetes-model-autoscaling-6.12.1.jar
+  - io.fabric8.kubernetes-model-batch-6.12.1.jar
+  - io.fabric8.kubernetes-model-certificates-6.12.1.jar
+  - io.fabric8.kubernetes-model-common-6.12.1.jar
+  - io.fabric8.kubernetes-model-coordination-6.12.1.jar
+  - io.fabric8.kubernetes-model-core-6.12.1.jar
+  - io.fabric8.kubernetes-model-discovery-6.12.1.jar
+  - io.fabric8.kubernetes-model-events-6.12.1.jar
+  - io.fabric8.kubernetes-model-extensions-6.12.1.jar
+  - io.fabric8.kubernetes-model-flowcontrol-6.12.1.jar
+  - io.fabric8.kubernetes-model-gatewayapi-6.12.1.jar
+  - io.fabric8.kubernetes-model-metrics-6.12.1.jar
+  - io.fabric8.kubernetes-model-networking-6.12.1.jar
+  - io.fabric8.kubernetes-model-node-6.12.1.jar
+  - io.fabric8.kubernetes-model-policy-6.12.1.jar
+  - io.fabric8.kubernetes-model-rbac-6.12.1.jar
+  - io.fabric8.kubernetes-model-resource-6.12.1.jar
+  - io.fabric8.kubernetes-model-scheduling-6.12.1.jar
+  - io.fabric8.kubernetes-model-storageclass-6.12.1.jar
+  - io.fabric8.zjsonpatch-0.3.0.jar
   - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar
   - jakarta.inject.jakarta.inject-api-2.0.1.jar
   - jakarta.validation.jakarta.validation-api-3.0.2.jar
@@ -300,6 +329,7 @@ Scala/Java jars:
   - org.scala-lang.scala-reflect-2.13.18.jar
   - org.slf4j.jcl-over-slf4j-2.0.12.jar
   - org.slf4j.log4j-over-slf4j-2.0.12.jar
+  - org.snakeyaml.snakeyaml-engine-2.7.jar
   - org.yaml.snakeyaml-2.2.jar
 
 --------------------------------------------------------------------------------
@@ -327,7 +357,7 @@ Scala/Java jars:
   - net.sourceforge.argparse4j.argparse4j-0.9.0.jar
   - org.checkerframework.checker-qual-3.52.0.jar
   - org.slf4j.jul-to-slf4j-2.0.12.jar
-  - org.slf4j.slf4j-api-2.0.12.jar
+  - org.slf4j.slf4j-api-2.0.13.jar
 
 --------------------------------------------------------------------------------
 Dependencies under the BSD 3-Clause License

notebook-migration-service/build.sbt

@@ -83,5 +83,6 @@ libraryDependencies ++= Seq(
 libraryDependencies ++= Seq(
   "io.dropwizard" % "dropwizard-core" % dropwizardVersion,
   "io.dropwizard" % "dropwizard-auth" % dropwizardVersion, // Dropwizard Authentication module
-  "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.18.8"
+  "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.18.8",
+  "io.fabric8" % "kubernetes-client" % "6.12.1" // Provisions per-user JupyterLab pods
 )

notebook-migration-service/src/main/scala/org/apache/texera/service/NotebookMigrationService.scala

@@ -35,6 +35,7 @@ import org.apache.texera.dao.SqlServer
 import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature
 import java.nio.file.Path
 import org.apache.texera.service.resource.{HealthCheckResource, NotebookMigrationResource}
+import org.apache.texera.service.util.JupyterTokenDeriver
 
 class NotebookMigrationService
     extends Application[NotebookMigrationServiceConfiguration]
@@ -61,6 +62,9 @@ class NotebookMigrationService
       configuration: NotebookMigrationServiceConfiguration,
       environment: Environment
   ): Unit = {
+    // Refuse to boot a misconfigured per-user Jupyter rather than failing per request.
+    JupyterTokenDeriver.validateConfiguration()
+
     // Serve backend at /api
     environment.jersey.setUrlPattern("/api/*")