Stop ICMP port-unreachable reports from failing UDP receives on Windows

By default Windows reports an ICMP port-unreachable that answers any
datagram a UDP socket sent as WSAECONNRESET on that socket's next
receive. The UDP transport's socket is shared by every UDP peer, so one
unreachable peer address put receive errors, each logged as a warning
and counted, into the path every other peer's traffic arrives through.

Sockets the transport opens or adopts now have SIO_UDP_CONNRESET turned
off. If the ioctl is refused the socket still works as before, so the
failure is logged and the transport starts anyway.

windows-sys becomes a direct Windows dependency at the version socket2
and tokio already use, so the lock file gains no package.

Cargo.lock

@@ -1208,6 +1208,7 @@ dependencies = [
  "tracing-subscriber",
  "tun",
  "windows-service",
+ "windows-sys 0.61.2",
  "wintun",
  "zeroize",
 ]

Cargo.toml

@@ -78,6 +78,9 @@ bluer = { version = "0.17", features = ["bluetoothd", "l2cap"] }
 [target.'cfg(windows)'.dependencies]
 wintun = "0.5"
 windows-service = "0.8.1"
+# For WSAIoctl(SIO_UDP_CONNRESET) on the UDP socket. The version socket2 and
+# tokio already pull in, so the lock file gains no package.
+windows-sys = { version = "0.61", features = ["Win32_Networking_WinSock", "Win32_System_IO"] }
 
 [package.metadata.deb]
 maintainer = "Johnathan Corgan <johnathan@corganlabs.com>"

src/transport/udp/io/mod.rs

@@ -143,6 +143,43 @@ mod tests {
         assert!(reuse_port, "the adopted socket must carry SO_REUSEPORT");
     }
 
+    /// A datagram sent to a port nobody holds draws an ICMP port-unreachable.
+    /// Windows reports one, by default, as `WSAECONNRESET` on the socket's next
+    /// receive; this socket is shared by every peer, so the next datagram from
+    /// any of them must come through instead. Checked for both ways a socket
+    /// is set up.
+    #[cfg(windows)]
+    #[tokio::test]
+    async fn a_port_unreachable_answer_is_not_a_receive_error_on_windows() {
+        let opened = UdpRawSocket::open("127.0.0.1:0".parse().unwrap(), 65536, 65536)
+            .expect("failed to open");
+        let plain = std::net::UdpSocket::bind("127.0.0.1:0").expect("failed to bind");
+        let adopted = UdpRawSocket::adopt(plain, 65536, 65536).expect("failed to adopt");
+        for (how, raw) in [("open", opened), ("adopt", adopted)] {
+            let addr = raw.local_addr();
+            let sock = raw.into_async().expect("into_async");
+
+            let gone = std::net::UdpSocket::bind("127.0.0.1:0").expect("bind the closed port");
+            let gone_addr = gone.local_addr().expect("closed port address");
+            drop(gone);
+            sock.send_to(b"nobody", &gone_addr).await.expect("send_to");
+            // Let the port-unreachable arrive before the datagram that follows.
+            tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+
+            let peer = std::net::UdpSocket::bind("127.0.0.1:0").expect("bind the peer");
+            peer.send_to(b"peer", addr).expect("peer send");
+
+            let mut buf = [0u8; 16];
+            let got =
+                tokio::time::timeout(std::time::Duration::from_secs(5), sock.recv_from(&mut buf))
+                    .await
+                    .unwrap_or_else(|_| panic!("{how}: nothing received"));
+            let (n, from, _) = got.unwrap_or_else(|e| panic!("{how}: receive failed: {e}"));
+            assert_eq!(&buf[..n], b"peer", "{how}");
+            assert_eq!(from, peer.local_addr().unwrap(), "{how}");
+        }
+    }
+
     #[tokio::test]
     async fn test_async_udp_socket_send_recv() {
         let sock1 = UdpRawSocket::open("127.0.0.1:0".parse().unwrap(), 65536, 65536)

src/transport/udp/io/windows.rs

@@ -8,7 +8,10 @@
 use crate::transport::TransportError;
 use socket2::{Domain, Protocol, Socket, Type};
 use std::net::SocketAddr;
+use std::os::windows::io::AsRawSocket;
 use std::sync::Arc;
+use tracing::warn;
+use windows_sys::Win32::Networking::WinSock::{SIO_UDP_CONNRESET, SOCKET, SOCKET_ERROR, WSAIoctl};
 
 /// UDP socket wrapper (Windows).
 ///
@@ -44,6 +47,7 @@ impl UdpRawSocket {
 
         sock.bind(&bind_addr.into())
             .map_err(|e| TransportError::StartFailed(format!("bind failed: {}", e)))?;
+        ignore_port_unreachable(&sock);
 
         // Set socket buffer sizes
         sock.set_recv_buffer_size(recv_buf_size)
@@ -75,6 +79,7 @@ impl UdpRawSocket {
 
         sock.set_nonblocking(true)
             .map_err(|e| TransportError::StartFailed(format!("set nonblocking failed: {}", e)))?;
+        ignore_port_unreachable(&sock);
 
         sock.set_recv_buffer_size(recv_buf_size)
             .map_err(|e| TransportError::StartFailed(format!("set recv buffer: {}", e)))?;
@@ -126,6 +131,44 @@ impl UdpRawSocket {
     }
 }
 
+/// Stop an ICMP port-unreachable from failing the socket's next receive.
+///
+/// By default Windows reports a port-unreachable answering any datagram the
+/// socket sent as `WSAECONNRESET` on its next `recv_from`. The socket is shared
+/// by every peer, so one unreachable address would put errors into the receive
+/// loop that all the other peers' traffic arrives through. `SIO_UDP_CONNRESET`
+/// set to false turns the report off. A socket that refuses it still works and
+/// still sees the errors, so a failure is logged rather than returned.
+fn ignore_port_unreachable(sock: &Socket) {
+    let report: u32 = 0;
+    let mut returned: u32 = 0;
+    // SAFETY: the socket is open for the duration of the call, the input
+    // buffer is a live u32 of the stated size, no output buffer is passed, and
+    // the call is synchronous (no OVERLAPPED, no completion routine).
+    let rc = unsafe {
+        WSAIoctl(
+            sock.as_raw_socket() as SOCKET,
+            SIO_UDP_CONNRESET,
+            (&report as *const u32).cast(),
+            std::mem::size_of::<u32>() as u32,
+            std::ptr::null_mut(),
+            0,
+            &mut returned,
+            std::ptr::null_mut(),
+            None,
+        )
+    };
+    if rc == SOCKET_ERROR {
+        // Read before `warn!`, whose level and dispatcher checks run first and
+        // could replace the error code WSAIoctl left.
+        let err = std::io::Error::last_os_error();
+        warn!(
+            error = %err,
+            "Could not turn off ICMP port-unreachable reports on the UDP socket"
+        );
+    }
+}
+
 /// Async UDP socket wrapper (Windows).
 ///
 /// Uses `tokio::net::UdpSocket` directly. Kernel drop counting