fix: add socket to avoid network issue

.github/workflows/rust-ci.yaml

@@ -65,3 +65,29 @@ jobs:
 
       - name: Run E2E tests
         run: make test-e2e
+
+  test-e2e-tunnel:
+    name: Tunnel E2E Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Regression coverage for the userspace-networking + SOCKS5 mesh dial path
+      # (an unprivileged daemon reaching the controller only over the tailnet).
+      # The controller container needs /dev/net/tun, which GitHub's Linux runners
+      # provide; the daemon side is deliberately unprivileged.
+      - name: Run tunnel-mode E2E tests
+        run: make test-e2e-tunnel

Makefile

@@ -5,21 +5,22 @@ MATURIN := .venv/bin/maturin
 # Pinned so lint results don't shift when ruff changes its default rule set.
 RUFF_VERSION := ruff==0.15.15
 
-.PHONY: help build install dev test clean daemon-build daemon-release test-e2e docker-build docker-down
+.PHONY: help build install dev test clean daemon-build daemon-release test-e2e test-e2e-tunnel docker-build docker-down
 
 help:
 	@echo "SandD - Sandbox Daemon - Build Commands"
 	@echo ""
-	@echo "  make build          - Build Python package (debug mode)"
-	@echo "  make install        - Install Python package locally"
-	@echo "  make dev            - Install in development mode with hot reload"
-	@echo "  make test           - Run unit and integration tests (fast, no Docker)"
-	@echo "  make test-e2e       - Run end-to-end tests with Docker (slow)"
-	@echo "  make daemon-build   - Build daemon binary (debug)"
-	@echo "  make daemon-release - Build daemon binary (release)"
-	@echo "  make docker-build   - Build Docker image for daemon"
-	@echo "  make docker-down    - Stop and remove Docker containers"
-	@echo "  make clean          - Clean build artifacts"
+	@echo "  make build           - Build Python package (debug mode)"
+	@echo "  make install         - Install Python package locally"
+	@echo "  make dev             - Install in development mode with hot reload"
+	@echo "  make test            - Run unit and integration tests (fast, no Docker)"
+	@echo "  make test-e2e        - Run direct-mode end-to-end tests with Docker (slow)"
+	@echo "  make test-e2e-tunnel - Run tunnel-mode (Tailscale mesh) e2e tests (slow)"
+	@echo "  make daemon-build    - Build daemon binary (debug)"
+	@echo "  make daemon-release  - Build daemon binary (release)"
+	@echo "  make docker-build    - Build Docker image for daemon"
+	@echo "  make docker-down     - Stop and remove Docker containers"
+	@echo "  make clean           - Clean build artifacts"
 
 build: $(MATURIN)
 	$(MATURIN) build -m server/Cargo.toml
@@ -58,12 +59,26 @@ test-e2e: $(PYTEST) dev
 	@echo "Building Docker images..."
 	docker compose -f hack/docker/docker-compose.e2e.yml build
 	@echo ""
-	@echo "Running E2E tests with Docker..."
-	$(PYTEST) python/tests/ -m e2e -v -s
+	@echo "Running direct-mode E2E tests with Docker..."
+	$(PYTEST) python/tests/ -m "e2e and not tunnel" -v -s
 	@echo ""
 	@echo "Cleaning up containers..."
 	docker compose -f hack/docker/docker-compose.e2e.yml down
 
+# Tunnel-mode e2e uses its OWN compose stack (headscale + mesh) and the test
+# fixture mints the auth key mid-bringup, so it runs separately from test-e2e.
+# The `tunnel` marker selects only these tests; the fixture handles up/down of
+# docker-compose.tunnel-e2e.yml, but we `down` here too as a cleanup backstop.
+test-e2e-tunnel: $(PYTEST) dev
+	@echo "Building tunnel-mode Docker images..."
+	docker compose -f hack/docker/docker-compose.tunnel-e2e.yml build
+	@echo ""
+	@echo "Running tunnel-mode E2E tests (Tailscale/headscale mesh)..."
+	$(PYTEST) python/tests/ -m tunnel -v -s
+	@echo ""
+	@echo "Cleaning up containers..."
+	docker compose -f hack/docker/docker-compose.tunnel-e2e.yml down -v
+
 docker-build:
 	docker compose -f hack/docker/docker-compose.e2e.yml build
 

hack/docker/docker-compose.tunnel-e2e.yml

@@ -0,0 +1,140 @@
+# Tunnel-mode E2E — REGRESSION coverage for the userspace-networking dial path.
+#
+# WHY THIS EXISTS (the bug it guards against):
+#   The direct-mode e2e (docker-compose.e2e.yml) and examples/tunnel-simple both
+#   let the daemon reach the controller over the shared Docker BRIDGE (via the
+#   `controller` hostname / host.docker.internal). That means the tailnet is set
+#   up but NEVER load-bearing — a plain socket always had a route. So the one path
+#   Nebula actually uses in prod — an UNPRIVILEGED daemon in
+#   `--tun=userspace-networking`, where the tailnet has NO kernel route and the
+#   WebSocket must traverse tailscaled's SOCKS5 proxy — had zero coverage, and a
+#   regression there (daemons join the mesh yet `Active daemons: 0`) shipped.
+#
+# HOW THIS TEST FORCES THE MESH PATH:
+#   The daemon dials `ws://controller.sandd.local:8765/ws` — a MagicDNS name. The
+#   Docker embedded DNS only knows the container name `controller`, NOT the
+#   `.sandd.local` FQDN, so the bridge cannot resolve (let alone route) it. The
+#   name resolves ONLY inside tailscaled (MagicDNS) to the controller's 100.64.x
+#   mesh IP, and — because the daemon runs userspace-networking with no route to
+#   100.64.0.0/10 — the only way the WebSocket connects is THROUGH the SOCKS5 proxy
+#   with remote DNS (socks5h). Without that wiring in sandd, this test hangs at
+#   "daemon failed to connect"; with it, the daemon connects and exec works.
+#
+# BOTH SIDES RUN USERSPACE-NETWORKING (Server(connect="tunnel") and sandd --tunnel
+# each start `tailscaled --tun=userspace-networking`). The controller keeps
+# NET_ADMIN + /dev/net/tun only as a harmless fallback / to mirror the infra side
+# of Nebula; in userspace mode tailscaled forwards inbound mesh TCP to the local
+# :8765 listener, so no kernel TUN is actually required. The daemon is deliberately
+# UNPRIVILEGED (no NET_ADMIN/TUN) — that is the TENANT side (the GPU workload
+# container) and the exact constraint this test proves works.
+#
+# Orchestrated by python/tests/test_e2e_tunnel.py (mints the auth key between
+# `up headscale` and `up controller daemon`). Not part of the default e2e run.
+
+services:
+  # Headscale coordination server — assigns mesh IPs, runs MagicDNS for
+  # *.sandd.local (see base_domain in the example's headscale-config.yaml).
+  headscale:
+    image: headscale/headscale:0.23
+    command: serve
+    volumes:
+      # Reuse the example's config verbatim: magic_dns: true, base_domain:
+      # sandd.local — that base_domain is what makes controller.sandd.local resolve.
+      - ../../examples/tunnel-simple/headscale-config.yaml:/etc/headscale/config.yaml:ro
+      - headscale-data:/var/lib/headscale
+    networks:
+      - mesh
+    environment:
+      - TZ=UTC
+
+  # Controller — the SandD server, on the mesh. Loops list_daemons()+exec and
+  # prints distinctive markers the test greps for. NET_ADMIN/TUN: infra side.
+  controller:
+    hostname: controller
+    build:
+      context: ../..
+      dockerfile: hack/docker/Dockerfile.server-tunnel
+    command:
+      - /bin/bash
+      - -c
+      - |
+        set -e
+        # Do NOT run tailscaled/`tailscale up` here: Server(connect="tunnel")
+        # brings up its own tailscaled (--tun=userspace-networking) and runs
+        # `tailscale up` internally (server/src/lib.rs setup_tunnel_controller).
+        # A manual `tailscale up` first would set --hostname, and the Server's
+        # second `up` (which omits it) then fails tailscale's "must mention all
+        # non-default flags" guard. The tailnet hostname comes from the OS
+        # hostname instead, which compose sets via `hostname: controller` below
+        # -> MagicDNS name controller.sandd.local.
+        python3 -u << 'PYEOF'
+        import os, time
+        from sandd import Server, TunnelConfig
+        cfg = TunnelConfig(authkey=os.environ["SANDD_TUNNEL_AUTH_KEY"],
+                           server="http://headscale:8080")
+        # Server joins the mesh (userspace-networking) and listens on :8765;
+        # tailscale proxies inbound mesh connections to this local listener.
+        server = Server(host="0.0.0.0", port=8765, connect="tunnel", tunnel_config=cfg)
+        print("[ctrl] controller ready, waiting for daemons", flush=True)
+        seen = set()
+        while True:
+            # list_daemons() returns DaemonInfo objects; exec() and the log
+            # markers want the plain id string (d.id), not the object repr.
+            for d in server.list_daemons():
+                did = d.id
+                if did not in seen:
+                    seen.add(did)
+                    print(f"[ctrl] DAEMON_CONNECTED {did}", flush=True)
+                r = server.exec(did, "hostname")
+                if r.success:
+                    print(f"[ctrl] EXEC_OK {did} {r.stdout.strip()}", flush=True)
+            time.sleep(2)
+        PYEOF
+    environment:
+      - SANDD_TUNNEL_AUTH_KEY=${SANDD_TUNNEL_AUTH_KEY:-}
+    cap_add:
+      - NET_ADMIN
+    devices:
+      - /dev/net/tun
+    depends_on:
+      - headscale
+    networks:
+      - mesh
+
+  # Daemon — the TENANT side. UNPRIVILEGED, userspace-networking. Dials the
+  # controller by its MagicDNS name so the bridge cannot carry the connection.
+  daemon:
+    build:
+      context: ../..
+      dockerfile: hack/docker/Dockerfile.daemon-tunnel
+    entrypoint: ["/bin/bash", "-c"]
+    command:
+      - |
+        set -e
+        echo "[daemon] starting sandd --tunnel (userspace-networking, unprivileged)"
+        # sandd itself brings up tailscaled --tun=userspace-networking with the
+        # SOCKS5 proxy, joins the mesh, and dials the controller THROUGH the proxy.
+        # The MagicDNS name is resolved remotely by tailscaled (socks5h), never on
+        # the Docker bridge — so this only succeeds over the mesh.
+        exec sandd \
+          --server-url=ws://controller.sandd.local:8765/ws \
+          --daemon-id=tunnel-daemon-1 \
+          --tunnel \
+          --tunnel-authkey="${SANDD_TUNNEL_AUTH_KEY:-}" \
+          --tunnel-server=http://headscale:8080
+    environment:
+      - SANDD_TUNNEL_AUTH_KEY=${SANDD_TUNNEL_AUTH_KEY:-}
+      - RUST_LOG=info
+    # NO cap_add, NO devices — this is the whole point: it must work unprivileged.
+    depends_on:
+      - headscale
+      - controller
+    networks:
+      - mesh
+
+volumes:
+  headscale-data:
+
+networks:
+  mesh:
+    driver: bridge

pyproject.toml

@@ -55,4 +55,5 @@ exclude = [
 asyncio_mode = "auto"
 markers = [
     "e2e: end-to-end tests with Docker (slow, skip by default)",
+    "tunnel: tunnel-mode (Tailscale/headscale) e2e; needs its own compose stack",
 ]