From 7365094cc5befb11fae787bf300fcbaa2720ce95 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 3 Aug 2026 15:25:44 +0100 Subject: [PATCH 1/4] fix: add socket to avoid network issue Signed-off-by: kerthcet --- .github/workflows/rust-ci.yaml | 26 ++++ Cargo.lock | 13 ++ Makefile | 23 +++- hack/docker/docker-compose.tunnel-e2e.yml | 137 +++++++++++++++++++ pyproject.toml | 1 + python/tests/test_e2e_tunnel.py | 154 ++++++++++++++++++++++ sandd/Cargo.toml | 7 + sandd/src/main.rs | 82 +++++++++++- 8 files changed, 437 insertions(+), 6 deletions(-) create mode 100644 hack/docker/docker-compose.tunnel-e2e.yml create mode 100644 python/tests/test_e2e_tunnel.py diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 76a81a5..4adfb17 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -65,3 +65,29 @@ jobs: - name: Run E2E tests run: make test-e2e + + test-e2e-tunnel: + name: Tunnel E2E Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Regression coverage for the userspace-networking + SOCKS5 mesh dial path + # (an unprivileged daemon reaching the controller only over the tailnet). + # The controller container needs /dev/net/tun, which GitHub's Linux runners + # provide; the daemon side is deliberately unprivileged. + - name: Run tunnel-mode E2E tests + run: make test-e2e-tunnel diff --git a/Cargo.lock b/Cargo.lock index edce33c..c223fc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1717,6 +1717,7 @@ dependencies = [ "tempfile", "tokio", "tokio-process", + "tokio-socks", "tokio-tungstenite", "tokio-util", "tracing", @@ -2208,6 +2209,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tokio-socks" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7e2948f60dbe26b35f2c7fb74ac2854c1fddded0fe9d7548fcc674a246f7615" +dependencies = [ + "either", + "futures-util", + "thiserror", + "tokio", +] + [[package]] name = "tokio-sync" version = "0.1.8" diff --git a/Makefile b/Makefile index 52ad972..906a3f4 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ MATURIN := .venv/bin/maturin # Pinned so lint results don't shift when ruff changes its default rule set. RUFF_VERSION := ruff==0.15.15 -.PHONY: help build install dev test clean daemon-build daemon-release test-e2e docker-build docker-down +.PHONY: help build install dev test clean daemon-build daemon-release test-e2e test-e2e-tunnel docker-build docker-down help: @echo "SandD - Sandbox Daemon - Build Commands" @@ -14,7 +14,8 @@ help: @echo " make install - Install Python package locally" @echo " make dev - Install in development mode with hot reload" @echo " make test - Run unit and integration tests (fast, no Docker)" - @echo " make test-e2e - Run end-to-end tests with Docker (slow)" + @echo " make test-e2e - Run direct-mode end-to-end tests with Docker (slow)" + @echo " make test-e2e-tunnel- Run tunnel-mode (Tailscale mesh) e2e tests (slow)" @echo " make daemon-build - Build daemon binary (debug)" @echo " make daemon-release - Build daemon binary (release)" @echo " make docker-build - Build Docker image for daemon" @@ -58,12 +59,26 @@ test-e2e: $(PYTEST) dev @echo "Building Docker images..." docker compose -f hack/docker/docker-compose.e2e.yml build @echo "" - @echo "Running E2E tests with Docker..." - $(PYTEST) python/tests/ -m e2e -v -s + @echo "Running direct-mode E2E tests with Docker..." + $(PYTEST) python/tests/ -m "e2e and not tunnel" -v -s @echo "" @echo "Cleaning up containers..." docker compose -f hack/docker/docker-compose.e2e.yml down +# Tunnel-mode e2e uses its OWN compose stack (headscale + mesh) and the test +# fixture mints the auth key mid-bringup, so it runs separately from test-e2e. +# The `tunnel` marker selects only these tests; the fixture handles up/down of +# docker-compose.tunnel-e2e.yml, but we `down` here too as a cleanup backstop. +test-e2e-tunnel: $(PYTEST) dev + @echo "Building tunnel-mode Docker images..." + docker compose -f hack/docker/docker-compose.tunnel-e2e.yml build + @echo "" + @echo "Running tunnel-mode E2E tests (Tailscale/headscale mesh)..." + $(PYTEST) python/tests/ -m tunnel -v -s + @echo "" + @echo "Cleaning up containers..." + docker compose -f hack/docker/docker-compose.tunnel-e2e.yml down -v + docker-build: docker compose -f hack/docker/docker-compose.e2e.yml build diff --git a/hack/docker/docker-compose.tunnel-e2e.yml b/hack/docker/docker-compose.tunnel-e2e.yml new file mode 100644 index 0000000..46f16e4 --- /dev/null +++ b/hack/docker/docker-compose.tunnel-e2e.yml @@ -0,0 +1,137 @@ +# Tunnel-mode E2E — REGRESSION coverage for the userspace-networking dial path. +# +# WHY THIS EXISTS (the bug it guards against): +# The direct-mode e2e (docker-compose.e2e.yml) and examples/tunnel-simple both +# let the daemon reach the controller over the shared Docker BRIDGE (via the +# `controller` hostname / host.docker.internal). That means the tailnet is set +# up but NEVER load-bearing — a plain socket always had a route. So the one path +# Nebula actually uses in prod — an UNPRIVILEGED daemon in +# `--tun=userspace-networking`, where the tailnet has NO kernel route and the +# WebSocket must traverse tailscaled's SOCKS5 proxy — had zero coverage, and a +# regression there (daemons join the mesh yet `Active daemons: 0`) shipped. +# +# HOW THIS TEST FORCES THE MESH PATH: +# The daemon dials `ws://controller.sandd.local:8765/ws` — a MagicDNS name. The +# Docker embedded DNS only knows the container name `controller`, NOT the +# `.sandd.local` FQDN, so the bridge cannot resolve (let alone route) it. The +# name resolves ONLY inside tailscaled (MagicDNS) to the controller's 100.64.x +# mesh IP, and — because the daemon runs userspace-networking with no route to +# 100.64.0.0/10 — the only way the WebSocket connects is THROUGH the SOCKS5 proxy +# with remote DNS (socks5h). Without that wiring in sandd, this test hangs at +# "daemon failed to connect"; with it, the daemon connects and exec works. +# +# ASYMMETRY (mirrors Nebula, deliberately): +# - controller: NET_ADMIN + /dev/net/tun. It is the INFRA side (in-cluster), and +# accepting INBOUND mesh connections on a normal listening socket needs a real +# TUN. This is not the constrained side, so privilege here is fine. +# - daemon: UNPRIVILEGED, userspace-networking. This is the TENANT side (the GPU +# workload container) — exactly what must work without NET_ADMIN/TUN. +# +# Orchestrated by python/tests/test_e2e_tunnel.py (mints the auth key between +# `up headscale` and `up controller daemon`). Not part of the default e2e run. + +services: + # Headscale coordination server — assigns mesh IPs, runs MagicDNS for + # *.sandd.local (see base_domain in the example's headscale-config.yaml). + headscale: + image: headscale/headscale:0.23 + command: serve + volumes: + # Reuse the example's config verbatim: magic_dns: true, base_domain: + # sandd.local — that base_domain is what makes controller.sandd.local resolve. + - ../../examples/tunnel-simple/headscale-config.yaml:/etc/headscale/config.yaml:ro + - headscale-data:/var/lib/headscale + networks: + - mesh + environment: + - TZ=UTC + + # Controller — the SandD server, on the mesh. Loops list_daemons()+exec and + # prints distinctive markers the test greps for. NET_ADMIN/TUN: infra side. + controller: + hostname: controller + build: + context: ../.. + dockerfile: hack/docker/Dockerfile.server-tunnel + command: + - /bin/bash + - -c + - | + set -e + echo "[ctrl] starting tailscaled (kernel TUN, infra side)" + tailscaled --state=/var/lib/tailscale/tailscaled.state & + sleep 3 + tailscale up \ + --authkey="${SANDD_TUNNEL_AUTH_KEY:-}" \ + --login-server=http://headscale:8080 \ + --hostname=controller \ + --accept-routes + echo "[ctrl] mesh IP: $$(tailscale ip -4)" + python3 -u << 'PYEOF' + import os, time + from sandd import Server, TunnelConfig + cfg = TunnelConfig(authkey=os.environ["SANDD_TUNNEL_AUTH_KEY"], + server="http://headscale:8080") + # The server joins the mesh via the running tailscaled/TUN above and + # listens on :8765 across it. + server = Server(host="0.0.0.0", port=8765, connect="tunnel", tunnel_config=cfg) + print("[ctrl] controller ready, waiting for daemons", flush=True) + seen = set() + while True: + for did in server.list_daemons(): + if did not in seen: + seen.add(did) + print(f"[ctrl] DAEMON_CONNECTED {did}", flush=True) + r = server.exec(did, "hostname") + if r.success: + print(f"[ctrl] EXEC_OK {did} {r.stdout.strip()}", flush=True) + time.sleep(2) + PYEOF + environment: + - SANDD_TUNNEL_AUTH_KEY=${SANDD_TUNNEL_AUTH_KEY:-} + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun + depends_on: + - headscale + networks: + - mesh + + # Daemon — the TENANT side. UNPRIVILEGED, userspace-networking. Dials the + # controller by its MagicDNS name so the bridge cannot carry the connection. + daemon: + build: + context: ../.. + dockerfile: hack/docker/Dockerfile.daemon-tunnel + entrypoint: ["/bin/bash", "-c"] + command: + - | + set -e + echo "[daemon] starting sandd --tunnel (userspace-networking, unprivileged)" + # sandd itself brings up tailscaled --tun=userspace-networking with the + # SOCKS5 proxy, joins the mesh, and dials the controller THROUGH the proxy. + # The MagicDNS name is resolved remotely by tailscaled (socks5h), never on + # the Docker bridge — so this only succeeds over the mesh. + exec sandd \ + --server-url=ws://controller.sandd.local:8765/ws \ + --daemon-id=tunnel-daemon-1 \ + --tunnel \ + --tunnel-authkey="${SANDD_TUNNEL_AUTH_KEY:-}" \ + --tunnel-server=http://headscale:8080 + environment: + - SANDD_TUNNEL_AUTH_KEY=${SANDD_TUNNEL_AUTH_KEY:-} + - RUST_LOG=info + # NO cap_add, NO devices — this is the whole point: it must work unprivileged. + depends_on: + - headscale + - controller + networks: + - mesh + +volumes: + headscale-data: + +networks: + mesh: + driver: bridge diff --git a/pyproject.toml b/pyproject.toml index 15f0979..acbf21b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,4 +55,5 @@ exclude = [ asyncio_mode = "auto" markers = [ "e2e: end-to-end tests with Docker (slow, skip by default)", + "tunnel: tunnel-mode (Tailscale/headscale) e2e; needs its own compose stack", ] diff --git a/python/tests/test_e2e_tunnel.py b/python/tests/test_e2e_tunnel.py new file mode 100644 index 0000000..66cb2b6 --- /dev/null +++ b/python/tests/test_e2e_tunnel.py @@ -0,0 +1,154 @@ +"""Tunnel-mode E2E — regression test for the userspace-networking dial path. + +Run with: make test-e2e-tunnel + +WHY A SEPARATE FILE FROM test_e2e.py: + test_e2e.py is DIRECT mode: daemons reach the host's :8765 over the Docker + bridge. That (and examples/tunnel-simple) let a plain socket carry the + connection, so the tailnet was set up but never load-bearing. The path Nebula + uses in prod — an UNPRIVILEGED daemon in --tun=userspace-networking, where the + tailnet has no kernel route and the WebSocket must go THROUGH tailscaled's + SOCKS5 proxy — had zero coverage. This test exercises exactly that path. + +HOW IT FORCES THE MESH (see docker-compose.tunnel-e2e.yml for the full rationale): + The daemon dials the controller by its MagicDNS name (controller.sandd.local), + which the Docker bridge cannot resolve — only tailscaled can. Combined with the + daemon being unprivileged/userspace-networking, the ONLY way it connects is via + the SOCKS5 proxy. Pre-fix, the daemon joins the mesh but never connects, and + this test fails; post-fix it connects and exec works. + +The controller runs INSIDE a container (it needs a real TUN, awkward for a host +Python process), so we assert on the controller's log markers rather than driving +a Server() from the test process. +""" + +import os +import subprocess +import time + +import pytest + +# Both markers: `e2e` (needs Docker, skipped in `make test`) and `tunnel` (needs +# the dedicated headscale/mesh compose stack — selected/excluded on its own). +pytestmark = [pytest.mark.e2e, pytest.mark.tunnel] + +COMPOSE_FILE = "hack/docker/docker-compose.tunnel-e2e.yml" +DAEMON_ID = "tunnel-daemon-1" +# Tunnel bring-up (tailscaled start + mesh join on two nodes + DERP negotiation) +# is much slower than direct mode; give it generous headroom. +CONNECT_TIMEOUT_S = 180 +POLL_INTERVAL_S = 3 + + +def _compose(*args, check=True, capture=True): + return subprocess.run( + ["docker", "compose", "-f", COMPOSE_FILE, *args], + check=check, + capture_output=capture, + text=True, + ) + + +def _headscale_container() -> str: + """Resolve the headscale container id (compose name is project-prefixed).""" + out = _compose("ps", "-q", "headscale").stdout.strip() + assert out, "headscale container not found — did `compose up headscale` run?" + return out.splitlines()[0] + + +def _controller_logs() -> str: + # check=False: the controller may still be starting; empty logs are fine. + return _compose("logs", "controller", check=False).stdout + + +@pytest.fixture(scope="module") +def tunnel_stack(): + """Bring up headscale, mint a reusable auth key, then start controller+daemon. + + The auth key must exist BEFORE the controller/daemon start (they consume it via + SANDD_TUNNEL_AUTH_KEY to join the mesh), so this mirrors the tunnel-simple + bring-up order: headscale first, mint key, then the rest. + """ + _compose("build") + + # 1. headscale only. + _compose("up", "-d", "headscale") + time.sleep(3) + + # 2. mint a reusable pre-auth key (controller + daemon share it in this test). + hs = _headscale_container() + subprocess.run( + ["docker", "exec", hs, "headscale", "users", "create", "sandd"], + check=False, # idempotent-ish: fine if the user already exists on rerun + capture_output=True, + text=True, + ) + key = subprocess.run( + [ + "docker", "exec", hs, "headscale", "preauthkeys", "create", + "--user", "sandd", "--reusable", "--expiration", "1h", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip().splitlines()[-1].strip() + assert key and " " not in key, f"unexpected preauthkey output: {key!r}" + + # 3. controller + daemon, with the freshly-minted key in their env. Compose + # substitutes ${SANDD_TUNNEL_AUTH_KEY} from this process's environment. + subprocess.run( + ["docker", "compose", "-f", COMPOSE_FILE, "up", "-d", "controller", "daemon"], + check=True, + capture_output=True, + text=True, + env={**os.environ, "SANDD_TUNNEL_AUTH_KEY": key}, + ) + + yield + + _compose("down", "-v", check=False) + + +def _wait_for_marker(marker: str, timeout: float) -> str: + """Poll the controller logs until `marker` appears; return the logs seen.""" + deadline = time.time() + timeout + logs = "" + while time.time() < deadline: + logs = _controller_logs() + if marker in logs: + return logs + time.sleep(POLL_INTERVAL_S) + return logs + + +class TestE2ETunnel: + """The daemon must connect and be usable over the mesh, unprivileged.""" + + def test_daemon_connects_over_mesh(self, tunnel_stack): + """Daemon joins the mesh and its WebSocket reaches the controller. + + This is THE regression assertion: it can only pass if the daemon's + WebSocket traverses tailscaled's SOCKS5 proxy (userspace-networking has no + kernel route to the controller's mesh IP, and the MagicDNS name is + unresolvable on the Docker bridge). + """ + marker = f"DAEMON_CONNECTED {DAEMON_ID}" + logs = _wait_for_marker(marker, CONNECT_TIMEOUT_S) + if marker not in logs: + # Surface daemon logs too — a mesh-join vs SOCKS-dial failure shows here. + daemon_logs = _compose("logs", "daemon", check=False).stdout + pytest.fail( + "daemon never connected to the controller over the mesh within " + f"{CONNECT_TIMEOUT_S}s (SOCKS5 tunnel path broken?).\n" + f"--- controller logs ---\n{logs[-3000:]}\n" + f"--- daemon logs ---\n{daemon_logs[-3000:]}" + ) + + def test_exec_over_mesh(self, tunnel_stack): + """A command runs on the daemon and its output returns over the mesh.""" + marker = f"EXEC_OK {DAEMON_ID}" + logs = _wait_for_marker(marker, CONNECT_TIMEOUT_S) + assert marker in logs, ( + "connected but exec over the mesh did not succeed; last controller " + f"logs:\n{logs[-3000:]}" + ) diff --git a/sandd/Cargo.toml b/sandd/Cargo.toml index 2b40b92..e30254f 100644 --- a/sandd/Cargo.toml +++ b/sandd/Cargo.toml @@ -45,6 +45,13 @@ uuid = { workspace = true } tokio-tungstenite = { version = "0.24", features = ["rustls-tls-native-roots"] } futures-util = "0.3" +# SOCKS5 client. In tunnel mode tailscaled runs with --tun=userspace-networking, +# so the tailnet has NO kernel route: a plain socket to a 100.64.x peer fails. We +# route the WebSocket THROUGH tailscaled's SOCKS5 proxy instead, and use remote +# DNS (socks5h) so MagicDNS names (e.g. sandd-controller.nebula.mesh) resolve +# inside tailscaled. Direct mode is untouched. +tokio-socks = "0.5" + # Process execution tokio-process = "0.2" diff --git a/sandd/src/main.rs b/sandd/src/main.rs index fc1f873..19ae5a2 100644 --- a/sandd/src/main.rs +++ b/sandd/src/main.rs @@ -15,6 +15,13 @@ use sysinfo::System; use tokio_tungstenite::tungstenite::protocol::Message as WsMessage; use tracing::{debug, error, info, warn}; +/// Address of the SOCKS5 proxy tailscaled exposes in tunnel mode (see +/// setup_tunnel). In --tun=userspace-networking there is no kernel route to the +/// tailnet, so the daemon dials the controller THROUGH this proxy to reach mesh +/// peers; using it with remote DNS also lets MagicDNS names resolve inside +/// tailscaled. Localhost-only: reachable solely by this container's daemon. +const TUNNEL_SOCKS_PROXY: &str = "127.0.0.1:1055"; + #[derive(Parser, Debug)] #[command(name = "sandd")] #[command( @@ -100,6 +107,7 @@ async fn main() -> Result<()> { &daemon_id, args.heartbeat_interval, labels.clone(), + args.tunnel, ) .await { @@ -117,6 +125,7 @@ async fn connect_and_serve( daemon_id: &str, heartbeat_interval: u64, labels: HashMap, + tunnel: bool, ) -> Result<()> { info!("Connecting to server at {}", server_url); @@ -128,6 +137,47 @@ async fn connect_and_serve( tokio_tungstenite::tungstenite::http::HeaderValue::from_static("sandd.v1"), ); + // Two transports, ONE serve loop (generic over the stream): + // - tunnel mode: the tailnet has no kernel route in userspace-networking, so + // open the TCP hop THROUGH tailscaled's SOCKS5 proxy, then run the + // WebSocket over that socket. The target host is passed to the proxy as a + // name (not pre-resolved locally), so MagicDNS names resolve inside + // tailscaled ("socks5h" semantics). + // - direct mode: unchanged — dial the server straight with connect_async. + if tunnel { + let uri = request.uri().clone(); + let host = uri + .host() + .ok_or_else(|| anyhow::anyhow!("tunnel: server URL has no host: {}", server_url))? + .to_string(); + // ws:// -> 80, wss:// -> 443 if unspecified; Nebula sets an explicit :8765. + let port = uri.port_u16().unwrap_or(match uri.scheme_str() { + Some("wss") => 443, + _ => 80, + }); + + info!( + "Dialing {}:{} via tailscaled SOCKS5 proxy {}", + host, port, TUNNEL_SOCKS_PROXY + ); + // (host, port) with a non-IP host becomes a SOCKS "domain" target, so + // tailscaled does the DNS — this is what makes MagicDNS names work. + let socks = tokio_socks::tcp::Socks5Stream::connect(TUNNEL_SOCKS_PROXY, (host.as_str(), port)) + .await + .with_context(|| { + format!( + "tunnel: failed to reach {}:{} through SOCKS5 proxy {} (is tailscaled up and joined?)", + host, port, TUNNEL_SOCKS_PROXY + ) + })?; + + let (ws_stream, response) = tokio_tungstenite::client_async_tls(request, socks) + .await + .context("tunnel: WebSocket handshake over SOCKS5 failed")?; + log_negotiated_protocol(&response); + return serve(ws_stream, daemon_id, heartbeat_interval, labels).await; + } + let (ws_stream, response) = match tokio_tungstenite::connect_async(request).await { Ok(result) => result, Err(e) => { @@ -135,14 +185,33 @@ async fn connect_and_serve( return Err(anyhow::anyhow!("Failed to connect to server: {}", e)); } }; + log_negotiated_protocol(&response); + serve(ws_stream, daemon_id, heartbeat_interval, labels).await +} - // Check negotiated protocol +/// Log the WebSocket subprotocol the server negotiated (shared by both transports). +fn log_negotiated_protocol( + response: &tokio_tungstenite::tungstenite::handshake::client::Response, +) { if let Some(protocol) = response.headers().get("sec-websocket-protocol") { info!("Negotiated protocol: {:?}", protocol); } else { warn!("Server did not negotiate protocol"); } +} +/// Run the daemon session over an established WebSocket stream. Generic over the +/// transport so the direct (connect_async) and tunnel (SOCKS5) paths share one +/// implementation. +async fn serve( + ws_stream: tokio_tungstenite::WebSocketStream, + daemon_id: &str, + heartbeat_interval: u64, + labels: HashMap, +) -> Result<()> +where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, +{ info!("WebSocket connection established"); let (mut ws_tx, mut ws_rx) = ws_stream.split(); @@ -612,9 +681,18 @@ async fn setup_tunnel(args: &Args) -> Result<()> { info!("Starting tailscaled..."); - // Start tailscaled in background (if not already running) + // Start tailscaled in background (if not already running). + // + // --socks5-server is what makes tunnel mode actually work: with + // --tun=userspace-networking there is no TUN device and thus no kernel route + // to the tailnet (100.64.0.0/10), so a plain socket to the controller's mesh + // address always fails. The SOCKS5 proxy is the entry point INTO tailscaled's + // userspace network stack; connect_and_serve dials the controller through it + // (see TUNNEL_SOCKS_PROXY) so the WebSocket rides the mesh. Bound to localhost + // so only this container's daemon can use it. let _tailscaled = Command::new("tailscaled") .arg("--tun=userspace-networking") + .arg(format!("--socks5-server={}", TUNNEL_SOCKS_PROXY)) .arg("--state=/var/lib/tailscale/tailscaled.state") .spawn() .context("Failed to start tailscaled")?; From bc62a6a5ca17112da46b74834a0c25942dc155e2 Mon Sep 17 00:00:00 2001 From: Kante Yin Date: Mon, 3 Aug 2026 15:33:02 +0100 Subject: [PATCH 2/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- sandd/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandd/src/main.rs b/sandd/src/main.rs index 19ae5a2..4a31437 100644 --- a/sandd/src/main.rs +++ b/sandd/src/main.rs @@ -681,7 +681,7 @@ async fn setup_tunnel(args: &Args) -> Result<()> { info!("Starting tailscaled..."); - // Start tailscaled in background (if not already running). + // Start tailscaled in background. // // --socks5-server is what makes tunnel mode actually work: with // --tun=userspace-networking there is no TUN device and thus no kernel route From 9af06e8486dfb1e607ede53b6acc1e2b2daa53dc Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 3 Aug 2026 15:33:45 +0100 Subject: [PATCH 3/4] format Signed-off-by: kerthcet --- Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 906a3f4..ec8b5bb 100644 --- a/Makefile +++ b/Makefile @@ -10,17 +10,17 @@ RUFF_VERSION := ruff==0.15.15 help: @echo "SandD - Sandbox Daemon - Build Commands" @echo "" - @echo " make build - Build Python package (debug mode)" - @echo " make install - Install Python package locally" - @echo " make dev - Install in development mode with hot reload" - @echo " make test - Run unit and integration tests (fast, no Docker)" - @echo " make test-e2e - Run direct-mode end-to-end tests with Docker (slow)" - @echo " make test-e2e-tunnel- Run tunnel-mode (Tailscale mesh) e2e tests (slow)" - @echo " make daemon-build - Build daemon binary (debug)" - @echo " make daemon-release - Build daemon binary (release)" - @echo " make docker-build - Build Docker image for daemon" - @echo " make docker-down - Stop and remove Docker containers" - @echo " make clean - Clean build artifacts" + @echo " make build - Build Python package (debug mode)" + @echo " make install - Install Python package locally" + @echo " make dev - Install in development mode with hot reload" + @echo " make test - Run unit and integration tests (fast, no Docker)" + @echo " make test-e2e - Run direct-mode end-to-end tests with Docker (slow)" + @echo " make test-e2e-tunnel - Run tunnel-mode (Tailscale mesh) e2e tests (slow)" + @echo " make daemon-build - Build daemon binary (debug)" + @echo " make daemon-release - Build daemon binary (release)" + @echo " make docker-build - Build Docker image for daemon" + @echo " make docker-down - Stop and remove Docker containers" + @echo " make clean - Clean build artifacts" build: $(MATURIN) $(MATURIN) build -m server/Cargo.toml From 615a0ff115262886800a25f4a0a263c9bdf1eef4 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 3 Aug 2026 16:10:57 +0100 Subject: [PATCH 4/4] fix test Signed-off-by: kerthcet --- hack/docker/docker-compose.tunnel-e2e.yml | 39 ++++++++++++----------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/hack/docker/docker-compose.tunnel-e2e.yml b/hack/docker/docker-compose.tunnel-e2e.yml index 46f16e4..c99b25e 100644 --- a/hack/docker/docker-compose.tunnel-e2e.yml +++ b/hack/docker/docker-compose.tunnel-e2e.yml @@ -20,12 +20,13 @@ # with remote DNS (socks5h). Without that wiring in sandd, this test hangs at # "daemon failed to connect"; with it, the daemon connects and exec works. # -# ASYMMETRY (mirrors Nebula, deliberately): -# - controller: NET_ADMIN + /dev/net/tun. It is the INFRA side (in-cluster), and -# accepting INBOUND mesh connections on a normal listening socket needs a real -# TUN. This is not the constrained side, so privilege here is fine. -# - daemon: UNPRIVILEGED, userspace-networking. This is the TENANT side (the GPU -# workload container) — exactly what must work without NET_ADMIN/TUN. +# BOTH SIDES RUN USERSPACE-NETWORKING (Server(connect="tunnel") and sandd --tunnel +# each start `tailscaled --tun=userspace-networking`). The controller keeps +# NET_ADMIN + /dev/net/tun only as a harmless fallback / to mirror the infra side +# of Nebula; in userspace mode tailscaled forwards inbound mesh TCP to the local +# :8765 listener, so no kernel TUN is actually required. The daemon is deliberately +# UNPRIVILEGED (no NET_ADMIN/TUN) — that is the TENANT side (the GPU workload +# container) and the exact constraint this test proves works. # # Orchestrated by python/tests/test_e2e_tunnel.py (mints the auth key between # `up headscale` and `up controller daemon`). Not part of the default e2e run. @@ -58,27 +59,29 @@ services: - -c - | set -e - echo "[ctrl] starting tailscaled (kernel TUN, infra side)" - tailscaled --state=/var/lib/tailscale/tailscaled.state & - sleep 3 - tailscale up \ - --authkey="${SANDD_TUNNEL_AUTH_KEY:-}" \ - --login-server=http://headscale:8080 \ - --hostname=controller \ - --accept-routes - echo "[ctrl] mesh IP: $$(tailscale ip -4)" + # Do NOT run tailscaled/`tailscale up` here: Server(connect="tunnel") + # brings up its own tailscaled (--tun=userspace-networking) and runs + # `tailscale up` internally (server/src/lib.rs setup_tunnel_controller). + # A manual `tailscale up` first would set --hostname, and the Server's + # second `up` (which omits it) then fails tailscale's "must mention all + # non-default flags" guard. The tailnet hostname comes from the OS + # hostname instead, which compose sets via `hostname: controller` below + # -> MagicDNS name controller.sandd.local. python3 -u << 'PYEOF' import os, time from sandd import Server, TunnelConfig cfg = TunnelConfig(authkey=os.environ["SANDD_TUNNEL_AUTH_KEY"], server="http://headscale:8080") - # The server joins the mesh via the running tailscaled/TUN above and - # listens on :8765 across it. + # Server joins the mesh (userspace-networking) and listens on :8765; + # tailscale proxies inbound mesh connections to this local listener. server = Server(host="0.0.0.0", port=8765, connect="tunnel", tunnel_config=cfg) print("[ctrl] controller ready, waiting for daemons", flush=True) seen = set() while True: - for did in server.list_daemons(): + # list_daemons() returns DaemonInfo objects; exec() and the log + # markers want the plain id string (d.id), not the object repr. + for d in server.list_daemons(): + did = d.id if did not in seen: seen.add(did) print(f"[ctrl] DAEMON_CONNECTED {did}", flush=True)