Merge pull request #454 from 9seconds/tcp-notsent-lowat

Add TCP_NOTSENT_LOWAT setting

network/v2/init.go

@@ -55,6 +55,14 @@ const (
 	// tcpLingerTimeout defines a number of seconds to wait for sending
 	// unacknowledged data.
 	tcpLingerTimeout = 1
+
+	// tcpNotSentLowat limits the amount of unsent data queued in the
+	// kernel write buffer per socket. When the unsent data drops below
+	// this threshold, the socket becomes writable again. This reduces
+	// per-connection memory usage and bufferbloat by applying
+	// back-pressure to the relay loop instead of piling up data in
+	// kernel buffers.
+	tcpNotSentLowat = 128 * 1024
 )
 
 var (

network/v2/sockopts.go

@@ -25,6 +25,7 @@ func setCommonSocketOptions(conn *net.TCPConn, keepAliveConfig net.KeepAliveConf
 
 	setCongestionControl(rawConn)
 	setTCPUserTimeout(rawConn, keepAliveConfig)
+	setNotSentLowat(rawConn)
 
 	return nil
 }

network/v2/sockopts_congestion.go

@@ -0,0 +1,20 @@
+//go:build linux
+
+package network
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// setCongestionControl sets BBR as the TCP congestion control algorithm.
+// BBR provides better throughput over lossy and high-latency links compared
+// to the default cubic, which is especially beneficial for mobile and
+// home internet clients. This is best-effort: silently ignored if the
+// kernel does not have tcp_bbr available.
+func setCongestionControl(conn syscall.RawConn) {
+	conn.Control(func(fd uintptr) { //nolint: errcheck
+		unix.SetsockoptString(int(fd), unix.IPPROTO_TCP, unix.TCP_CONGESTION, "bbr") //nolint: errcheck
+	})
+}

network/v2/sockopts_congestion_stub.go

@@ -0,0 +1,7 @@
+//go:build !linux
+
+package network
+
+import "syscall"
+
+func setCongestionControl(conn syscall.RawConn) {}

network/v2/sockopts_lowat.go

@@ -0,0 +1,20 @@
+//go:build linux || darwin
+
+package network
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// setNotSentLowat sets TCP_NOTSENT_LOWAT which limits the amount of
+// unsent data queued in the kernel write buffer. Once unsent data drops
+// below this threshold the socket becomes writable again, applying
+// back-pressure to the relay loop instead of piling up data in kernel
+// buffers. This reduces per-connection memory and bufferbloat.
+func setNotSentLowat(conn syscall.RawConn) {
+	conn.Control(func(fd uintptr) { //nolint: errcheck
+		unix.SetsockoptInt(int(fd), unix.IPPROTO_TCP, unix.TCP_NOTSENT_LOWAT, tcpNotSentLowat) //nolint: errcheck
+	})
+}

network/v2/sockopts_lowat_stub.go

@@ -0,0 +1,7 @@
+//go:build !linux && !darwin
+
+package network
+
+import "syscall"
+
+func setNotSentLowat(conn syscall.RawConn) {}

network/v2/sockopts_reuseaddr.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package network
 

network/v2/sockopts_reuseaddr_stub.go

@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package network
 

network/v2/sockopts_usertimeout.go

@@ -46,13 +46,3 @@ func setTCPUserTimeout(conn syscall.RawConn, cfg net.KeepAliveConfig) {
 		unix.SetsockoptInt(int(fd), unix.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(timeout.Milliseconds())) //nolint: errcheck
 	})
 }
-
-func setCongestionControl(conn syscall.RawConn) {
-	conn.Control(func(fd uintptr) { //nolint: errcheck
-		// BBR provides better throughput over lossy and high-latency links compared
-		// to the default cubic, which is especially beneficial for mobile and
-		// home internet clients. This is best-effort: silently ignored if the
-		// kernel does not have tcp_bbr available.
-		unix.SetsockoptString(int(fd), unix.IPPROTO_TCP, unix.TCP_CONGESTION, "bbr") //nolint: errcheck
-	})
-}

network/v2/sockopts_usertimeout_stub.go

@@ -7,5 +7,4 @@ import (
 	"syscall"
 )
 
-func setCongestionControl(conn syscall.RawConn)                       {}
 func setTCPUserTimeout(conn syscall.RawConn, cfg net.KeepAliveConfig) {}