feat(notebook-migration, k8s): deploy the notebook migration tool on Kubernetes

access-control-service/src/main/scala/org/apache/texera/service/resource/AccessControlResource.scala

@@ -30,9 +30,9 @@ import org.apache.texera.auth.util.{ComputingUnitAccess, HeaderField}
 import org.apache.texera.common.config.{GuiConfig, LLMConfig}
 import org.apache.texera.dao.SqlServer
 import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum
-import org.apache.texera.dao.jooq.generated.tables.daos.WorkflowComputingUnitDao
+import org.apache.texera.dao.jooq.generated.tables.daos.{UserJupyterDao, WorkflowComputingUnitDao}
 
-import java.net.URLDecoder
+import java.net.{URI, URLDecoder}
 import java.nio.charset.StandardCharsets
 import java.util.Optional
 import scala.jdk.CollectionConverters.{CollectionHasAsScala, MapHasAsScala}
@@ -51,6 +51,10 @@ object AccessControlResource extends LazyLogging {
   private val pvePvesCuidPath: Regex = """^/?(?:auth/)?(?:api/|wsapi/)?pve/pves/([0-9]+)$""".r
   private val pvePackagesCuidPath: Regex =
     """^/?(?:auth/)?(?:api/|wsapi/)?pve/([0-9]+)/[^/]+/packages/.+$""".r
+  // Per-user JupyterLab. The uid is in the path because a browser cannot attach Texera
+  // credentials to the requests Jupyter's own scripts make, so it is the only place the
+  // owner can be read from.
+  private val jupyterPath: Regex = """^/?(?:auth/)?jupyter/([0-9]+)(?:/.*)?$""".r
 
   /**
     * Authorize the request based on the path and headers.
@@ -68,6 +72,7 @@ object AccessControlResource extends LazyLogging {
     logger.info(s"Authorizing request for path: $path")
 
     path match {
+      case jupyterPath(uid) => routeToJupyter(uid)
       case wsapiWorkflowWebsocket() | apiExecutionsStats() | apiExecutionsResultExport() |
           pveRoute() =>
         checkComputingUnitAccess(uriInfo, headers, bodyOpt)
@@ -77,6 +82,39 @@ object AccessControlResource extends LazyLogging {
     }
   }
 
+  /**
+    * Resolve which JupyterLab pod a request belongs to. This routes; it does not authorize.
+    *
+    * Jupyter is loaded in an iframe and then issues its own requests for assets, contents and
+    * kernel websockets. None of those can carry a Texera token, and there is no session cookie
+    * to fall back on, so the caller cannot be authenticated per request. What protects one
+    * user's notebooks from another is the per-user Jupyter token, which is derived from a
+    * server-held secret and is unguessable; reaching the right pod without it yields a 403 from
+    * Jupyter itself. Cross-pod traffic is blocked separately by a NetworkPolicy.
+    */
+  private def routeToJupyter(uid: String): Response = {
+    val recordedUrl =
+      try {
+        val dao = new UserJupyterDao(SqlServer.getInstance().createDSLContext().configuration())
+        Option(dao.fetchOneByUid(uid.toInt)).map(_.getInternalUrl)
+      } catch {
+        case e: Exception =>
+          logger.error(s"Failed to look up the Jupyter registered for user $uid", e)
+          return Response.status(Response.Status.FORBIDDEN).build()
+      }
+
+    // Envoy routes on an authority, so the scheme and the base path are stripped back off the
+    // address the provisioner recorded.
+    recordedUrl.map(url => new URI(url).getAuthority).filter(a => a != null && a.nonEmpty) match {
+      case Some(authority) =>
+        logger.info(s"Routing Jupyter for user $uid to recorded host: $authority")
+        Response.ok().header("Host", authority).build()
+      case None =>
+        logger.warn(s"Refusing Jupyter for user $uid: no Jupyter is registered")
+        Response.status(Response.Status.FORBIDDEN).build()
+    }
+  }
+
   private def checkComputingUnitAccess(
       uriInfo: UriInfo,
       headers: HttpHeaders,

access-control-service/src/test/scala/org/apache/texera/AccessControlResourceSpec.scala

@@ -27,11 +27,13 @@ import org.apache.texera.dao.jooq.generated.enums.{
   WorkflowComputingUnitTypeEnum
 }
 import org.apache.texera.dao.jooq.generated.tables.daos.{
+  UserJupyterDao,
   ComputingUnitUserAccessDao,
   UserDao,
   WorkflowComputingUnitDao
 }
 import org.apache.texera.dao.jooq.generated.tables.pojos.{
+  UserJupyter,
   ComputingUnitUserAccess,
   User,
   WorkflowComputingUnit
@@ -73,6 +75,11 @@ class AccessControlResourceSpec
   private val testNoAccessRecordedUri: String =
     "computing-unit-6.compute-unit-svc.default.svc.cluster.local:7777"
 
+  // What the provisioner records for a user's Jupyter: scheme, authority and the base path
+  // the pod serves under. Only the authority may reach Envoy as a Host header.
+  private val testJupyterInternalUrl: String =
+    "http://jupyter-1.jupyter-svc.texera-jupyter-pool.svc.cluster.local:8888/jupyter/1"
+
   private val testUser1: User = {
     val user = new User()
     user.setUid(1)
@@ -190,6 +197,14 @@ class AccessControlResourceSpec
     readOnlyAccess.setPrivilege(PrivilegeEnum.READ)
     computingUnitOfUserDao.insert(readOnlyAccess)
 
+    // Per-user Jupyter: user 1 has one registered, user 2 deliberately does not.
+    val jupyterDao = new UserJupyterDao(getDSLContext.configuration())
+    val jupyter = new UserJupyter()
+    jupyter.setUid(testUser1.getUid)
+    jupyter.setInternalUrl(testJupyterInternalUrl)
+    jupyter.setPublicUrl("https://texera.example.com/jupyter/1")
+    jupyterDao.insert(jupyter)
+
     token = JwtAuth.jwtToken(JwtAuth.jwtClaims(testUser1))
     token2 = JwtAuth.jwtToken(JwtAuth.jwtClaims(testUser2))
   }
@@ -722,4 +737,56 @@ class AccessControlResourceSpec
     response.getStatus shouldBe Response.Status.OK.getStatusCode
     response.getHeaderString("Host") shouldBe testRecordedUri
   }
+
+  // -- per-user JupyterLab routing --------------------------------------------
+
+  it should "route a Jupyter request to the pod recorded for the uid in the path" in {
+    val (uri, headers) = mockRequest("/jupyter/1/notebooks/work/notebook.ipynb", None)
+    val response = new AccessControlResource().authorizeGet(uri, headers)
+
+    response.getStatus shouldBe Response.Status.OK.getStatusCode
+    // The scheme and base path are stripped: Envoy routes on an authority alone.
+    response.getHeaderString("Host") shouldBe
+      "jupyter-1.jupyter-svc.texera-jupyter-pool.svc.cluster.local:8888"
+  }
+
+  it should "route Jupyter's own subrequests, which carry no token" in {
+    // The iframe's asset and API calls cannot present Texera credentials, so routing has to
+    // work without one. The per-user Jupyter token is what authorizes them.
+    val (uri, headers) =
+      mockRequest("/jupyter/1/api/contents", None, authorizationHeader = None)
+    val response = new AccessControlResource().authorizeGet(uri, headers)
+
+    response.getStatus shouldBe Response.Status.OK.getStatusCode
+    response.getHeaderString("Host") should startWith("jupyter-1.")
+  }
+
+  it should "refuse a Jupyter request for a user with none registered" in {
+    val (uri, headers) = mockRequest("/jupyter/2/tree", None)
+    new AccessControlResource()
+      .authorizeGet(uri, headers)
+      .getStatus shouldBe Response.Status.FORBIDDEN.getStatusCode
+  }
+
+  it should "refuse a Jupyter request for a uid that does not exist" in {
+    val (uri, headers) = mockRequest("/jupyter/999999/tree", None)
+    new AccessControlResource()
+      .authorizeGet(uri, headers)
+      .getStatus shouldBe Response.Status.FORBIDDEN.getStatusCode
+  }
+
+  it should "not treat a Jupyter path without a uid as routable" in {
+    // Falls through to the catch-all, which denies.
+    val (uri, headers) = mockRequest("/jupyter/tree", None)
+    new AccessControlResource()
+      .authorizeGet(uri, headers)
+      .getStatus shouldBe Response.Status.FORBIDDEN.getStatusCode
+  }
+
+  it should "route the gateway-relative form of a Jupyter path" in {
+    val (uri, headers) = mockRequest("auth/jupyter/1/tree", None)
+    new AccessControlResource()
+      .authorizeGet(uri, headers)
+      .getStatus shouldBe Response.Status.OK.getStatusCode
+  }
 }

bin/k8s/templates/base/config-service/config-service-deployment.yaml

@@ -47,6 +47,11 @@ spec:
                 secretKeyRef:
                   name: {{ .Release.Name }}-postgresql
                   key: postgres-password
+            # Shows or hides the notebook migration tool in the workspace. Derived from the
+            # service's own toggle rather than listed in texeraEnvVars, so enabling the tool
+            # is one switch instead of two that can disagree.
+            - name: GUI_WORKFLOW_WORKSPACE_PYTHON_NOTEBOOK_MIGRATION_ENABLED
+              value: "{{ .Values.notebookMigrationService.enabled }}"
             {{- range .Values.texeraEnvVars }}
             - name: {{ .name }}
               value: "{{ .value }}"

bin/k8s/templates/base/gateway/gateway-routes.yaml

@@ -74,13 +74,26 @@ spec:
       backendRefs:
         - name: config-service-svc
           port: 9094
+    {{- if .Values.notebookMigrationService.enabled }}
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /api/notebook-migration
+      backendRefs:
+        - name: {{ .Values.notebookMigrationService.name }}-svc
+          port: {{ .Values.notebookMigrationService.service.port }}
+    {{- end }}
     - matches:
         - path:
             type: PathPrefix
             value: /api/models
         - path:
             type: PathPrefix
             value: /api/chat
+      {{- if and .Values.gatewayConfig .Values.gatewayConfig.llmRequestTimeout }}
+      timeouts:
+        request: {{ .Values.gatewayConfig.llmRequestTimeout | quote }}
+      {{- end }}
       backendRefs:
         - name: access-control-service-svc
           port: 9096
@@ -135,6 +148,13 @@ spec:
         - path:
             type: PathPrefix
             value: /api/pve
+        {{- if .Values.notebookMigrationService.enabled }}
+        # Per-user JupyterLab. ExtAuthz reads the uid from the path and rewrites Host to
+        # that user's pod; the per-user Jupyter token is what authorizes the request.
+        - path:
+            type: PathPrefix
+            value: /jupyter
+        {{- end }}
       backendRefs:
         - group: gateway.envoyproxy.io
           kind: Backend

bin/k8s/templates/base/jupyter-pool/jupyter-namespace.yaml

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+{{- if and .Values.notebookMigrationService.enabled .Values.jupyterPool.createNamespaces }}
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .Values.jupyterPool.namespace }}
+{{- end }}

bin/k8s/templates/base/jupyter-pool/jupyter-network-policy.yaml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+{{- if and .Values.notebookMigrationService.enabled .Values.jupyterPool.networkPolicy.enabled }}
+# Stops one user's JupyterLab from reaching another's. Users run arbitrary code in these
+# pods, so a neighbour in the pool is the one genuinely hostile caller. Allowing every
+# namespace but the pool's own denies pod-to-pod traffic inside it while leaving the real
+# callers working: the notebook migration service, and the Envoy proxy wherever the gateway
+# installation runs it.
+#
+# Defence in depth, not the authorisation boundary: the per-user Jupyter token is what stops
+# one user reading another's notebooks. Egress is left alone, since notebooks legitimately
+# install packages and call out.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ .Values.jupyterPool.name }}-deny-cross-user
+  namespace: {{ .Values.jupyterPool.namespace }}
+spec:
+  podSelector:
+    matchLabels:
+      type: jupyter
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        # kubernetes.io/metadata.name is set automatically on every namespace, so this
+        # selects "any namespace but the pool's own" without labelling anything by hand.
+        - namespaceSelector:
+            matchExpressions:
+              - key: kubernetes.io/metadata.name
+                operator: NotIn
+                values:
+                  - {{ .Values.jupyterPool.namespace }}
+{{- end }}

bin/k8s/templates/base/jupyter-pool/jupyter-prepull-daemonset.yaml

@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+{{- if and .Values.notebookMigrationService.enabled .Values.jupyterPool.prepullImage }}
+# Pulls the JupyterLab image onto every node ahead of time. Pods are created on demand and
+# the service waits a bounded time for one to answer, so a first-time pull on a cold node can
+# outlast that wait and the provisioning attempt is discarded. Mirrors the computing unit
+# pool's prepuller. Set jupyterPool.prepullImage to false to trade cold starts for one fewer
+# pod per node.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ .Release.Name }}-jupyter-prepuller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-jupyter-prepuller
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-jupyter-prepuller
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-jupyter-prepuller
+    spec:
+      restartPolicy: Always
+      tolerations:
+        - operator: "Exists"
+      initContainers:
+        - name: prepuller
+          image: {{ .Values.texera.imageRegistry }}/{{ .Values.jupyterPool.imageName }}:{{ .Values.texera.imageTag }}
+          imagePullPolicy: {{ .Values.texeraImages.pullPolicy }}
+          command: ["sh", "-c", "true"]
+      containers:
+        - name: pause
+          image: gcr.io/google_containers/pause:3.2
+          resources:
+            limits:
+              cpu: 1m
+              memory: 8Mi
+            requests:
+              cpu: 1m
+              memory: 8Mi
+{{- end }}

bin/k8s/templates/base/jupyter-pool/jupyter-resource-quota.yaml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+{{- if and .Values.notebookMigrationService.enabled .Values.jupyterPool.createNamespaces .Values.jupyterPool.maxRequestedResources }}
+# Ceiling for the pool as a whole. Pods are created on demand, one per user, so without this
+# a busy deployment has no upper bound on what the tool can consume.
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: {{ .Values.jupyterPool.name }}-resource-quota
+  namespace: {{ .Values.jupyterPool.namespace }}
+spec:
+  hard:
+    requests.cpu: "{{ .Values.jupyterPool.maxRequestedResources.cpu }}"
+    requests.memory: {{ .Values.jupyterPool.maxRequestedResources.memory }}
+{{- end }}