diff --git a/launch/core/platforms/linux.py b/launch/core/platforms/linux.py index bd87786..2ded871 100644 --- a/launch/core/platforms/linux.py +++ b/launch/core/platforms/linux.py @@ -17,6 +17,7 @@ ) import os, json +import posixpath from typing import Any, Optional import queue import threading @@ -205,9 +206,15 @@ def apply_patch(self, patch: str, verbose: bool = False) -> bool: filename = f"{uuid.uuid4()}.diff" hostpath = os.path.join(self.mnt_host, filename) - with open(hostpath, "w") as f: + # Keep patch files LF-only when the evaluator runs on Windows; + # otherwise git inside the Linux container receives CRLF hunks. + with open(hostpath, "w", encoding="utf-8", newline="\n") as f: f.write(patch) - containerpath = os.path.join(self.mnt_container, filename) + # Container paths are Linux/POSIX paths even when the evaluator is + # launched from Windows. os.path.join would use backslashes on + # Windows and turn /mnt_tmp/ into /mnt_tmp\, which bash + # interprets as /mnt_tmp. Use posixpath for in-container paths. + containerpath = posixpath.join(self.mnt_container, filename) cmd = f"""git apply --reject --whitespace=nowarn {containerpath} """ res = self.send_command(cmd) diff --git a/launch/core/platforms/windows.py b/launch/core/platforms/windows.py index 28c146e..e14bf3e 100644 --- a/launch/core/platforms/windows.py +++ b/launch/core/platforms/windows.py @@ -17,10 +17,25 @@ from launch.core.platforms.linux import LinuxRuntime import os +import base64 +import re from typing import Any import queue import uuid + +# `fakerepo` is used by Go integration tests as a deliberately unreachable +# local registry. It must bypass the evaluator's outbound proxy so callers see +# the direct socket error the test asserts, rather than a proxy-generated EOF. +DEFAULT_WINDOWS_CONTAINER_NO_PROXY = "localhost,127.0.0.1,::1,fakerepo" + + +def get_windows_container_no_proxy() -> str: + """Return the container bypass list while preserving an explicit override.""" + return os.environ.get( + "SWE_WINDOWS_CONTAINER_NO_PROXY", DEFAULT_WINDOWS_CONTAINER_NO_PROXY + ) + import docker from docker.models.containers import Container @@ -48,6 +63,7 @@ def __init__( params={"stdin": 1, "stdout": 1, "stderr": 1, "stream": 1} ) self.output_queue: queue.Queue[bytes] = queue.Queue() + self.capture_token = uuid.uuid4().hex self.stopped = False self._start_output_thread() self._clear_initial_prompt() @@ -76,50 +92,226 @@ def __init__( Write-Output "###PS1END###" "PS $wd> " } +try { + $raw = $Host.UI.RawUI + $raw.BufferSize = New-Object System.Management.Automation.Host.Size(32767, 3000) + $raw.WindowSize = New-Object System.Management.Automation.Host.Size(240, 60) +} catch { + # Some Windows Docker/ConPTY hosts expose a read-only or absent RawUI. + # Prompt metadata remains valid without resizing the console buffer. +} ''') self.preparation_commands = [] def send_command(self, command: str, timeout: int|None = None) -> CommandResult: - ''' - timeout: deprecated arg for backward compatibility. In minute. If not specified use self.timeout from object inittialization. - ''' - timeout = self.command_timeout * 60 if timeout is None else timeout * 60 # in seconds - - if self.stopped: - raise RuntimeError("container is stopped. Currently we have not enabled container restart after docker commit. If you need to restore the container you must launch from the new image you committed.") - - # Normalize newline semantics for interactive shells - # For PowerShell, ensure CRLF line endings - command = command.strip().replace("\r\n", "\n").replace("\n", "\r\n") - # Add extra CRLF for multi-line blocks to signal completion - command += "\r\n\r\nprompt\r\n\r\n" - - self._clear_initial_prompt() - - self._send_bytes(command.encode()) - - output, metadata = self._read_raw_output(timeout=timeout) - if metadata is not None: - return CommandResult(output=output, metadata=metadata) + """Execute a command through Docker's non-interactive exec API. - # handle timeout - # to kill the task completely, should Ctrl^C for several times - for _ in range(10): - self._send_bytes(b"\x03") - - kill_timeout = 5 - kill_output, kill_metadata = self._read_raw_output(timeout=kill_timeout) + Windows containers expose a ConPTY when started with ``tty=True``. The + old persistent attach path could leave long-running commands in an idle + PowerShell prompt and only produced zero-byte capture files. Direct + ``exec_run`` avoids ConPTY and returns the command output as a normal + process result while keeping the existing runtime API unchanged. + """ + timeout_seconds = self.command_timeout * 60 if timeout is None else timeout * 60 - output = output + kill_output + "\n**Exited due to timeout**\n" - if kill_metadata is not None: - kill_metadata.exit_code = TIMEOUT_EXIT_CODE - return CommandResult(output=output, metadata=kill_metadata) + if self.stopped: + raise RuntimeError( + "container is stopped. Currently we have not enabled container restart after docker commit. " + "If you need to restore the container you must launch from the new image you committed." + ) - fallback_metadata = CmdOutputMetadata( - exit_code=TIMEOUT_EXIT_CODE, + import json + command_name = f"windows-command-{uuid.uuid4().hex}.ps1" + command_host_path = os.path.join(self.mnt_host, command_name) + command_guest_path = f"C:\\mnt_tmp\\{command_name}" + # Each exec is a fresh PowerShell process, so make the runtime working + # directory explicit rather than relying on persistent shell state. + script = ( + "Set-Location -LiteralPath 'C:\\testbed'\n" + + "$OutputEncoding = [Console]::OutputEncoding = New-Object System.Text.UTF8Encoding($false)\n" + + "$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'\n" + + command + + "\n" + + "$__slai_ec = if ($LASTEXITCODE -ne $null) { [int]$LASTEXITCODE } elseif ($?) { 0 } else { 1 }\n" + + "Write-Output '###PS1JSON###'\n" + + "$__slai_obj=[ordered]@{exit_code=$__slai_ec; username=$env:USERNAME; " + + "hostname=$env:COMPUTERNAME; working_dir=(Get-Location).Path; py_interpreter_path=''}\n" + + "$__slai_obj | ConvertTo-Json -Compress\n" + + "Write-Output '###PS1END###'\n" ) + os.makedirs(self.mnt_host, exist_ok=True) + with open(command_host_path, "w", encoding="utf-8", newline="\n") as fh: + fh.write(script) - return CommandResult(output=output, metadata=fallback_metadata) + try: + # exec_run is deliberately non-TTY: no ConPTY wrapping, no prompt + # state, and no second capture file are involved. + result = self.container.exec_run( + [ + "powershell.exe", "-NoLogo", "-NoProfile", "-NonInteractive", + "-ExecutionPolicy", "Bypass", "-File", command_guest_path, + ], + workdir=self.working_dir, + stdout=True, + stderr=True, + demux=False, + ) + raw_output = result.output + if isinstance(raw_output, tuple): + raw_output = (raw_output[0] or b"") + (raw_output[1] or b"") + raw_bytes = raw_output or b"" + # Windows PowerShell redirection uses UTF-16LE by default. Decode + # the BOM-marked stream before stripping ANSI/control characters; + # UTF-8 replacement turns every second byte into NULs and makes the + # Go JSONL parser silently lose the test records. + if raw_bytes.startswith((b"\xff\xfe", b"\xfe\xff")): + output = raw_bytes.decode("utf-16", errors="replace") + else: + output = raw_bytes.decode("utf-8", errors="replace") + output = ANSI_ESCAPE.sub("", output).replace("\r", "") + matches = CmdOutputMetadata.matches_ps1_metadata(output) + metadata = CmdOutputMetadata.from_ps1_match(matches[-1]) if matches else None + if metadata is not None: + output = output[:matches[-1].start()] + else: + exit_code = result.exit_code + if exit_code is None: + exit_code = TIMEOUT_EXIT_CODE if timeout_seconds <= 0 else 1 + metadata = CmdOutputMetadata( + exit_code=int(exit_code), + username=None, + hostname=None, + working_dir=self.working_dir, + py_interpreter_path=None, + ) + return CommandResult(output=output, metadata=metadata) + finally: + try: + os.remove(command_host_path) + except OSError: + pass + + def apply_patch(self, patch: str, verbose: bool = False) -> bool: + """Apply a unified diff using a native Windows container path.""" + output_temp = "\n\n<<<<<>>>>>\n\n" + filename = f"{uuid.uuid4()}.diff" + hostpath = os.path.join(self.mnt_host, filename) + guestpath = f"C:\\mnt_tmp\\{filename}" + os.makedirs(self.mnt_host, exist_ok=True) + with open(hostpath, "w", encoding="utf-8", newline="\n") as fh: + fh.write(patch) + try: + # LinuxRuntime.apply_patch uses /mnt_tmp, which is invalid inside a + # Windows container. Keep the same patch protocol but use the + # mounted Windows path explicitly. + res = self.send_command( + f"git apply --reject --whitespace=nowarn '{guestpath}'" + ) + ok = int(res.metadata.exit_code) == 0 + if ok: + if verbose: + print(f"git apply {guestpath} ---- Patch applied Successfully!", flush=True) + return True + if verbose: + print(output_temp.format(out=res.output), flush=True) + return False + finally: + try: + os.remove(hostpath) + except OSError: + pass + + @staticmethod + def _repair_wrapped_output(output: str) -> str: + """Repair hard-wraps emitted by Windows Docker/ConPTY for JSONL output. + + The Windows pseudo-console can split long JSON records at a fixed column, + including in the middle of keys and escaped strings. This conservative + repair only joins lines that are clearly continuations of a JSON record; + it leaves ordinary test output and prompt lines unchanged. + """ + if not output or '{"Time":' not in output: + return output + lines = output.replace("\r\n", "\n").replace("\r", "\n").split("\n") + repaired = [] + current = "" + for line in lines: + if not line: + continue + if line.startswith('{"Time":'): + if current: + repaired.append(current) + current = line + continue + if current.startswith('{"Time":'): + current += line + if line.endswith('}'): + repaired.append(current) + current = "" + continue + if current: + repaired.append(current) + current = line + if current: + repaired.append(current) + return "\n".join(repaired) + ("\n" if output.endswith("\n") else "") + + @staticmethod + def _repair_structured_go_json(output: str) -> str: + """Recover JSONL records after ConPTY hard-wraps and duplicated letters. + + The Windows host path uses an interactive PowerShell/ConPTY transcript. A + long go-test JSON record can be split inside keys, package names, or test + names. The status object is still present in each record, so recover each + object and canonicalize only the fields used by the Windows evaluator. + """ + if not output or '{"Time":' not in output: + return output + records = [] + depth = 0 + in_string = False + escape = False + buf = [] + for ch in output: + if depth == 0: + if ch == '{': + depth = 1 + in_string = False + escape = False + buf = ['{'] + continue + buf.append(ch) + if in_string: + if escape: + escape = False + elif ch == '\\\\': + escape = True + elif ch == '"': + in_string = False + else: + if ch == '"': + in_string = True + elif ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + if depth == 0: + raw = ''.join(buf).replace('\\r', '').replace('\\n', '') + try: + import json + obj = json.loads(raw) + if obj.get('Action') in ('start', 'run', 'output', 'pass', 'fail', 'skip'): + for field in ('Package', 'Test'): + if isinstance(obj.get(field), str): + value = obj[field] + value = re.sub(r'\\s+', '', value) + value = re.sub(r'([A-Za-z])\\1+', r'\\1', value) + obj[field] = value + records.append(json.dumps(obj, ensure_ascii=False, separators=(',', ':'))) + except Exception: + pass + buf = [] + return '\\n'.join(records) + ('\\n' if records else '') @classmethod def _start_container( @@ -148,6 +340,10 @@ def _start_container( run_kwargs = { "cpu_count": CPU_CORES, # cpu_quota is Linux-only "mem_limit": MEM_LIMIT, + # Host Docker is already the Windows engine. Keep the default + # process isolation so SWE Windows runs directly on the host + # instead of adding the retired Hyper-V runner layer. + "isolation": "process", } container = client.containers.run( @@ -159,6 +355,22 @@ def _start_container( detach=True, environment={ "TERM": "xterm-mono", + # The Windows host in the CN environment cannot reliably reach + # proxy.golang.org. Keep module downloads deterministic inside + # SWE containers while allowing an outer environment override. + "GOPROXY": os.environ.get("GOPROXY", "https://goproxy.cn,direct"), + "GOTOOLCHAIN": os.environ.get("GOTOOLCHAIN", "local"), + # Container-side proxy is intentionally explicit. Docker Desktop's + # host proxy often appears as 127.0.0.1:7897, but inside a Windows + # container that points back to the container itself. For SWE runs + # set SWE_WINDOWS_CONTAINER_PROXY to the NAT gateway proxy, e.g. + # http://172.19.32.1:7897. + **({ + "HTTP_PROXY": os.environ.get("SWE_WINDOWS_CONTAINER_PROXY"), + "HTTPS_PROXY": os.environ.get("SWE_WINDOWS_CONTAINER_PROXY"), + "ALL_PROXY": os.environ.get("SWE_WINDOWS_CONTAINER_PROXY"), + "NO_PROXY": get_windows_container_no_proxy(), + } if os.environ.get("SWE_WINDOWS_CONTAINER_PROXY") else {}), }, working_dir=working_dir, extra_hosts=extra_hosts, diff --git a/tests/runtime_test.py b/tests/runtime_test.py index 809ec3d..8a9dd32 100644 --- a/tests/runtime_test.py +++ b/tests/runtime_test.py @@ -49,7 +49,11 @@ from launch.core.runtime import SetupRuntime from launch.core.platforms.linux import LinuxRuntime -from launch.core.platforms.windows import WindowsRuntime +from launch.core.platforms.windows import ( + DEFAULT_WINDOWS_CONTAINER_NO_PROXY, + WindowsRuntime, + get_windows_container_no_proxy, +) from launch.core.platforms.android import AndroidRuntime #from launch.core.platforms.macos import MacosRuntime