diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d974988..26c6bd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ concurrency: env: # Pin both dependencies so a moved result is about mere-ruby. A published # Mere improvement can be selected through workflow_dispatch. - MERE_REF: ${{ inputs.mere_ref || '01ed96805060232c47a839c11f62e9c464de603e' }} + MERE_REF: ${{ inputs.mere_ref || '991f299b20f18639713685fe3297e087ddc5b1d7' }} RUBY_SOURCE_REF: 3e93b23afcec3433f6aecc34b5d832799d754238 REF_RUBY_VERSION: "4.0.9" RUST_VERSION: "1.85.0" @@ -153,6 +153,8 @@ jobs: run: sh tools/psych_scalar_check.sh - name: Check IO read deadlines with an outer time limit run: python3 unittest/io_timeout_check.py + - name: Check addressed UDP with bounded children + run: python3 unittest/udp_datagram_check.py - name: Compare every corpus program run: sh run_corpus.sh - name: Check YAML store round trips diff --git a/KNOWN_GAPS.md b/KNOWN_GAPS.md index 0bd6431..64291da 100644 --- a/KNOWN_GAPS.md +++ b/KNOWN_GAPS.md @@ -1084,14 +1084,15 @@ on a chosen address and answers getsockname / getpeername itself: - **getsockname / getpeername are the kernel's own** (sock_local_addr / sock_peer_addr). They were read from outside -- lsof(8) on macOS, /proc on Linux, about 35 ms a socket -- before the runtime could ask. -- **UDP binds and connects, but cannot send to an address.** - `UDPSocket#bind` is bind(2), `#connect` / `send` / `recv` are the runtime's - udp_open / send / recv. `#send` to an address needs sendto(2) and raises - NotImplementedError, and a datagram's sender needs recvfrom(2): `#recvfrom` - on a socket that is not connected is Errno::ENOTCONN from getpeername(2), - which is why ipsocket/recvfrom_spec and socket/recvfrom_spec stay DIFF. A - datagram is one recv(2) / send(2) and never passes through the stream - buffer, so an empty one is `""` and is actually sent. +- **UDP supports addressed datagrams.** `UDPSocket#send` accepts host/port, + packed sockaddr, and Addrinfo destinations; `#recvfrom` reports each packet's + actual sender. The native sendto/recvfrom helpers preserve the socket's + descriptor and binding, pass flags to the kernel, and carry binary and empty + packets. `MSG_PEEK` leaves the datagram queued; short receives discard its + remainder. Zero-length receives retain the host kernel's behavior. + `UDPSocket#connect` still replaces the underlying descriptor, so preserving + a prior binding across connect remains unsupported. Generic `Socket` and + `IPSocket` datagram paths retain their existing limitations. - **UNIX sockets** need a sockaddr_un for connect(2) and bind(2) and an array for socketpair(2); they raise NotImplementedError. `Addrinfo.unix` and `Socket.sockaddr_un` are data and work. @@ -3568,7 +3569,7 @@ exactly, because the other half would have to invent something: | `cgi` — escapeHTML, unescapeHTML, the URI encoders, escapeElement | the request-shaped instance methods `#params`, `#header`, `#out`, `#[]`: they need a request environment, and this process is not a CGI script. ⚠ `CGI.new` itself SUCCEEDS and hands back an object none of them answer for -- the one row here that fails somewhere other than the name you asked for | | `io/console` — `StringIO#getch` | `#getpass`: ruby consumes one character more than "a line without the newline", and a method that is close but not the same is a wrong answer where a missing one is an honest refusal | | `openssl` — `OpenSSL::Digest`, HMAC, experimental AES CBC/ECB | TLS, X509, PKey: a gem should fail on the constant that names the missing capability, not load and fail somewhere else | -| `socket` — Addrinfo, Option, AncillaryData, constants; TCP in both families, connected UDP | bind(2)/sendto(2) to a chosen address and UNIX sockets EXIST and raise `NotImplementedError`, for the same reason (see the `socket` entry) | +| `socket` — Addrinfo, Option, AncillaryData, constants; TCP in both families, addressed UDP | `UDPSocket` supports addressed send/receive; generic socket datagram paths and binding preservation across UDP connect remain limited (see the `socket` entry) | ⚠ Each of these was added because something failed by NAME rather than by behaviour: `require "fcntl"` on line four of core/io/reopen_spec took that diff --git a/corpus/496_udp_datagrams.rb b/corpus/496_udp_datagrams.rb new file mode 100644 index 0000000..440a8f2 --- /dev/null +++ b/corpus/496_udp_datagrams.rb @@ -0,0 +1,205 @@ +require 'socket' + +[[Socket::AF_INET, '127.0.0.1'], [Socket::AF_INET6, '::1']].each do |family, host| + sockets = [] + begin + sender, first, second = Array.new(3) do + socket = UDPSocket.new(family) + sockets << socket + socket.bind(host, 0) + socket + end + source = sender.addr(false) + targets = [first, second] + ["a\0\xff".b, '', 'next'].each_with_index do |data, i| + receiver = targets[i % 2] + port = receiver.addr[1] + p sender.send(data, 0, host, port) + actual, address = receiver.recvfrom(100) + p [actual, actual.encoding, address == source, sender.addr(false) == source] + end + begin + sender.peeraddr + rescue => error + p error.class + end + # A packed sockaddr and Addrinfo both name a destination. + [Socket.sockaddr_in(first.addr[1], host), Addrinfo.udp(host, first.addr[1])].each do |destination| + p sender.send('packed', 0, destination) + data, address = first.recvfrom(32) + p [data, address == source] + end + p sender.send('abcdef', 0, host, first.addr[1]) + data, address = first.recvfrom(2, Socket::MSG_PEEK) + p [data, address == source] + p first.recv(4, Socket::MSG_PEEK) + p first.recvfrom(3).then { |data, address| [data, address == source] } + p sender.send('zero capacity', 0, host, first.addr[1]) + raise 'packet not ready' unless IO.select([first], nil, nil, 2) + p first.recvfrom(0).then { |data, address| [data, address == source] } + p sender.send('buffer', 0, host, first.addr[1]) + buffer = 'old'.encode(Encoding::UTF_16LE) + result = first.recv(3, 0, buffer) + p [result.equal?(buffer), result.bytes, result.encoding] + # Coercion happens once, before the native byte pointer is taken. + conversions = [] + message = Object.new + message.define_singleton_method(:to_str) { conversions << :message; 'coerced' } + flags = Object.new + flags.define_singleton_method(:to_int) { conversions << :flags; GC.start; 0 } + p sender.send(message, flags, host, first.addr[1]) + p [first.recv(32), conversions] + [nil, Object.new].each do |bad| + begin + sender.send(bad, 0, host, first.addr[1]) + rescue => error + p error.class + end + end + begin + first.recvfrom(-1) + rescue => error + p error.class + end + begin + sender.send('x' * 70_000, 0, host, first.addr[1]) + rescue => error + p error.class + end + [nil, 2**31, -2**31-1].each do |bad_flags| + begin + sender.send('bad flags', bad_flags, host, first.addr[1]) + rescue => error + p error.class + end + end + [2**31, -2**31-1].each do |bad_length| + begin + first.recvfrom(bad_length) + rescue => error + p error.class + end + end + p sender.send('outbuf', 0, host, first.addr[1]) + buffer = 'old' + data, address = first.recvfrom(64, nil, buffer) + p [data, data.equal?(buffer), address == source] + # Connected sends still use their peer; no fd replacement per send. + connected = UDPSocket.new(family) + sockets << connected + connected.connect(host, second.addr[1]) + peer = connected.peeraddr(false) + local = connected.addr(false) + p connected.send('connected', 0) + p second.recvfrom(32).then { |data, address| [data, address == local] } + p [connected.peeraddr(false) == peer, connected.addr(false) == local] + sender.close + begin + sender.send('closed', 0, host, first.addr[1]) + rescue => error + p error.class + end + ensure + sockets.each { |socket| socket.close unless socket.closed? } + end +end + +# The receiving thread must yield until another Ruby thread sends. +server = UDPSocket.new +client = UDPSocket.new +begin + server.bind('127.0.0.1', 0) + receiver = Thread.new { server.recvfrom(64) } + Thread.pass + client.send('thread', 0, '127.0.0.1', server.addr[1]) + data, address = receiver.value + p [data, address[1] == client.addr[1], address[3]] +ensure + server.close + client.close +end + +# User conversions can close the socket and immediately reuse its descriptor. +# A failed send must leave the destination empty; a failed receive must leave +# the replacement socket's queued packet intact. +[:message, :flags, :host, :port, :packed].each do |conversion| + receiver = UDPSocket.new + udp = UDPSocket.new + replacement = nil + begin + receiver.bind('127.0.0.1', 0) + fd = udp.fileno + port = receiver.addr[1] + packed = Socket.sockaddr_in(port, '127.0.0.1') + proxy = Object.new + proxy.define_singleton_method(conversion == :flags ? :to_int : :to_str) do + udp.close + replacement = UDPSocket.new + raise 'descriptor was not reused' unless replacement.fileno == fd + case conversion + when :message then 'unexpected' + when :flags then 0 + when :host then '127.0.0.1' + when :port then port.to_s + when :packed then packed + end + end + caught = nil + begin + if conversion == :packed + udp.send('unexpected', 0, proxy) + else + udp.send(conversion == :message ? proxy : 'unexpected', + conversion == :flags ? proxy : 0, + conversion == :host ? proxy : '127.0.0.1', + conversion == :port ? proxy : port) + end + rescue IOError => error + caught = error + end + raise 'send did not reject the closed stream' unless caught + raise 'send touched the replacement descriptor' if IO.select([receiver], nil, nil, 0.02) + p [:closed_during_send, conversion, caught.class, caught.message] + ensure + [udp, receiver, replacement].each { |socket| socket.close if socket && !socket.closed? } + end +end + +[:recv, :recvfrom].each do |method| + [:length, :flags].each do |conversion| + sender = UDPSocket.new + udp = UDPSocket.new + replacement = nil + begin + fd = udp.fileno + proxy = Object.new + proxy.define_singleton_method(:to_int) do + udp.close + replacement = UDPSocket.new + raise 'descriptor was not reused' unless replacement.fileno == fd + replacement.bind('127.0.0.1', 0) + sender.send('keep queued', 0, '127.0.0.1', replacement.addr[1]) + raise 'packet not ready' unless IO.select([replacement], nil, nil, 2) + conversion == :length ? 64 : 0 + end + caught = nil + begin + length = conversion == :length ? proxy : 64 + flags = conversion == :flags ? proxy : 0 + if method == :recv + udp.recv(length, flags) + else + udp.recvfrom(length, flags) + end + rescue IOError => error + caught = error + end + raise 'receive did not reject the closed stream' unless caught + raise 'receive consumed the replacement packet' unless IO.select([replacement], nil, nil, 0) + raise 'replacement packet changed' unless replacement.recv(64) == 'keep queued' + p [:closed_during_receive, method, conversion, caught.class, caught.message] + ensure + [udp, sender, replacement].each { |socket| socket.close if socket && !socket.closed? } + end + end +end diff --git a/m_eval_leaf.mere b/m_eval_leaf.mere index 0472eab..63d6686 100644 --- a/m_eval_leaf.mere +++ b/m_eval_leaf.mere @@ -6511,9 +6511,6 @@ and stdlib_src = fn (name: str) -> "end\n" ++ "\n" ++ "class UDPSocket < IPSocket\n" ++ - " # A datagram socket here can be CONNECTED -- udp_open is socket + connect --\n" ++ - " # and then send and receive; bind(2) and sendto(2) take a sockaddr, so a\n" ++ - " # socket that must receive on a chosen address is refused by name.\n" ++ " def initialize(family = Socket::AF_INET)\n" ++ " __new_block_warn if block_given?\n" ++ " fam = Socket::Constants.__family_arg(family)\n" ++ @@ -6539,24 +6536,56 @@ and stdlib_src = fn (name: str) -> " __move_to(fd)\n" ++ " 0\n" ++ " end\n" ++ - " def send(mesg, flags = 0, host = nil, port = nil)\n" ++ + " def send(mesg, flags, *dest)\n" ++ + " raise ArgumentError, \"wrong number of arguments\" if dest.length > 2\n" ++ " __check_open\n" ++ - " if host\n" ++ - " raise NotImplementedError, \"UDPSocket#send to an address is not implemented (sendto(2) takes a sockaddr)\"\n" ++ + " mesg = IO.__fd_str(mesg)\n" ++ + " flags = IO.__fd_cint(flags)\n" ++ + " if dest.length == 2\n" ++ + " ai = Addrinfo.__getaddrinfo(dest[0], Addrinfo.__port_of(dest[1], \"udp\"), @family, Socket::SOCK_DGRAM, 0, 0, false)[0]\n" ++ + " elsif dest.length == 1\n" ++ + " ai = dest[0].is_a?(Addrinfo) ? dest[0] : Addrinfo.new(IO.__fd_str(dest[0]))\n" ++ + " end\n" ++ + " host = ai ? ai.ip_address : nil\n" ++ + " port = ai ? ai.ip_port : 0\n" ++ + " loop do\n" ++ + " __check_open\n" ++ + " n = Socket.__udp_sendto(@__io_fd, mesg, flags, host, port)\n" ++ + " Addrinfo.__gai_error(\"sendto(2)\", 8) if n.nil?\n" ++ + " return n if n >= 0\n" ++ + " IO.__fd_raise(n, \"sendto(2)\") unless IO.__again?(n)\n" ++ + " IO.__sel_wait([[@__io_fd, 2, self]], -1)\n" ++ " end\n" ++ - " n = Socket.__udp_send(@__io_fd, mesg.to_s)\n" ++ - " raise Errno::EDESTADDRREQ, \"send(2)\" if n < 0\n" ++ - " n\n" ++ " end\n" ++ " def write(mesg)\n" ++ - " send(mesg)\n" ++ + " send(mesg, 0)\n" ++ " end\n" ++ - " def recv(maxlen, flags = 0, outbuf = nil)\n" ++ + " def __udp_packet(maxlen, flags)\n" ++ " __check_open\n" ++ - " __wait_readable\n" ++ - " d = Socket.__udp_recv(@__io_fd, maxlen)\n" ++ - " raise Errno::EAGAIN, \"recvfrom(2)\" if d.nil?\n" ++ - " outbuf ? outbuf.replace(d) : d\n" ++ + " flags = flags.nil? ? 0 : IO.__fd_cint(flags)\n" ++ + " maxlen = __fd_len!(IO.__fd_cint(maxlen))\n" ++ + " raise IOError, \"recv for buffered IO\" unless @__io_rbuf.empty?\n" ++ + " loop do\n" ++ + " __check_open\n" ++ + " packet = Socket.__udp_packet(@__io_fd, maxlen, flags)\n" ++ + " return packet unless packet.is_a?(Integer)\n" ++ + " IO.__fd_raise(packet, \"recvfrom(2)\") unless IO.__again?(packet)\n" ++ + " __wait_readable\n" ++ + " end\n" ++ + " end\n" ++ + " private :__udp_packet\n" ++ + " def recv(maxlen, flags = 0, outbuf = nil)\n" ++ + " outbuf = __fd_buf!(outbuf)\n" ++ + " data, _ = __udp_packet(maxlen, flags)\n" ++ + " outbuf ? __fd_into(outbuf, data) : data\n" ++ + " end\n" ++ + " def recvfrom(maxlen, flags = 0, outbuf = nil)\n" ++ + " outbuf = __fd_buf!(outbuf)\n" ++ + " data, address = __udp_packet(maxlen, flags)\n" ++ + " data = __fd_into(outbuf, data) if outbuf\n" ++ + " return [data, nil] if address.start_with?(\"other \")\n" ++ + " ai = Socket::Endpoint__.addrinfo(address, \"recvfrom(2)\", Socket::SOCK_DGRAM)\n" ++ + " [data, __ipaddr(ai, !@do_not_reverse_lookup)]\n" ++ " end\n" ++ "end\n" ++ "\n" ++ diff --git a/m_state.mere b/m_state.mere index 917ca96..6661d62 100644 --- a/m_state.mere +++ b/m_state.mere @@ -27,6 +27,9 @@ extern fn tcp_set_timeout: int -> int -> int; extern fn udp_open: str -> int -> int; extern fn udp_send: int -> int -> int -> int; extern fn udp_recv: int -> int -> int -> int; +// Addressed datagrams use the same byte arena and preserve the socket fd. +extern fn sock_sendto: int -> int -> int -> int -> str -> int -> int; +extern fn sock_recvfrom: int -> int -> int -> int -> str; // socket(2), listen(2) and shutdown(2) take and answer plain ints, and every // build that uses the tcp_* family above has in it, so these // three are ordinary libc externs -- the prototype Mere writes is the one the diff --git a/main.mere b/main.mere index e55306c..b0bf2b4 100644 --- a/main.mere +++ b/main.mere @@ -20851,6 +20851,40 @@ and invoke_val_c0 = fn env -> fn world -> fn blk -> fn (recv0: Val) -> fn (name0 (match args with | Cons (VInt fd, Cons (VStr h, _)) -> VInt (udp_put fd (sget h)) | _ -> VInt (0 - 1)) + // One datagram, with all conversions finished before touching the arena. + else if (str_len name == 12 && str_eq name "__udp_sendto") then + (match args with + | Cons (VInt fd, Cons (VStr h, Cons (VInt fl, Cons (host, Cons (VInt port, Nil))))) -> + (let data = sget h in + let n = str_len data in + if n > 65536 then raise_exc world "Errno::EMSGSIZE" "Message too long - sendto(2)" + else + (let p = tcp_arena () in + let _ = tcp_put_bytes p data 0 n in + let sent = (match host with VStr addr -> sock_sendto fd p n fl (sget addr) port + | _ -> sock_send fd p n fl) in + if sent >= 0 then VInt sent + else if sent == 0 - 2 && fd_last_errno () == 0 then VNil + else VInt (fd_neg_errno ()))) + | _ -> VInt (0 - 22)) + // Payload and sender are returned together, including for an empty packet. + else if (str_len name == 12 && str_eq name "__udp_packet") then + (match args with + | Cons (VInt fd, Cons (VInt cap, Cons (VInt fl, Nil))) -> + (let p = tcp_arena () in + let size = (if cap > 65536 then 65536 else cap) in + let t = sock_recvfrom fd p size fl in + if str_len t == 0 then VInt (fd_neg_errno ()) + else + (match str_split t " " with + | Cons (count, _) -> + (let n = int_of_str count in + let used = (if n > size then size else n) in + let data = mkstr (mem_to_str p used) in + let _ = (match data with VStr h -> senc_set h "ASCII-8BIT" | _ -> ()) in + arr_new (Cons (data, Cons (mkstr (substring t (str_len count + 1) (str_len t)), Nil)))) + | _ -> VInt (0 - 22))) + | _ -> VInt (0 - 22)) // send(2) with the flags as given (the platform's MSG_*): the count, // or -errno else if (str_len name == 11 && str_eq name "__send_flag") then diff --git a/unittest/udp_datagram_check.py b/unittest/udp_datagram_check.py new file mode 100644 index 0000000..5f38ea3 --- /dev/null +++ b/unittest/udp_datagram_check.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Compare addressed UDP behavior with a deadline on every child.""" +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parent.parent +reference = os.environ.get('REF_RUBY_BIN', 'ruby') +program = ROOT / 'corpus/496_udp_datagrams.rb' + + +def run(binary, env): + result = subprocess.run([str(binary), str(program)], env=env, + capture_output=True, timeout=15, check=True) + return result.stdout + + +expected = run(reference, os.environ.copy()) +for mode in ('on', 'off'): + env = os.environ.copy() + env.pop('MERE_RUBY_NO_HASH_INDEX', None) + if mode == 'off': + env['MERE_RUBY_NO_HASH_INDEX'] = '1' + actual = run(ROOT / 'mere-ruby', env) + assert actual == expected, f'UDP regression differs with hash index {mode}' + print(f'UDP datagrams: matches reference, hash index {mode}', flush=True)