From bed43c49995dbdf39e4720bd60503343a03f4fc1 Mon Sep 17 00:00:00 2001 From: Jakub Sitnicki Date: Wed, 30 Oct 2024 09:24:00 +0100 Subject: [PATCH] Add tests for configurable TIME-WAIT reuse delay This set of tests accompanies the kernel patch set which adds a knob for setting the TIME-WAIT reuse delay [1], which has been merged in [2]. Tests cover connection reincarnation through TIME-WAIT reuse with the default reuse delay (1 second) as well as with a reuse delay radically shortened down to 1 millisecond. We exercise both the happy and failure scenarios, where TW reuse is not possible because the required delay period has not elapsed yet. [1] https://lore.kernel.org/all/20241204-jakub-krn-909-poc-msec-tw-tstamp-v1-0-8b54467a0f34@cloudflare.com/ [2] https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=154dee7c3265bb8c1e9e87ee63dd195497155854 --- gtests/net/packetdrill/symbols_linux.c | 2 + .../fast_tw_reuse_fail_after_500usec.pkt | 51 ++++++++++++++++ .../tw_reuse/fast_tw_reuse_ok_after_1msec.pkt | 60 +++++++++++++++++++ .../tw_reuse/tw_reuse_fail_after_0sec.pkt | 46 ++++++++++++++ ...tw_reuse_fail_after_1sec_without_tsopt.pkt | 47 +++++++++++++++ .../tw_reuse/tw_reuse_fail_after_990msec.pkt | 47 +++++++++++++++ .../tw_reuse_ok_after_0sec_with_portlock.pkt | 58 ++++++++++++++++++ .../tw_reuse/tw_reuse_ok_after_1sec.pkt | 59 ++++++++++++++++++ 8 files changed, 370 insertions(+) create mode 100644 gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_fail_after_500usec.pkt create mode 100644 gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_ok_after_1msec.pkt create mode 100644 gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_0sec.pkt create mode 100644 gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_1sec_without_tsopt.pkt create mode 100644 gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_990msec.pkt create mode 100644 gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_0sec_with_portlock.pkt create mode 100644 gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_1sec.pkt diff --git a/gtests/net/packetdrill/symbols_linux.c b/gtests/net/packetdrill/symbols_linux.c index 16551438..43f5d33e 100644 --- a/gtests/net/packetdrill/symbols_linux.c +++ b/gtests/net/packetdrill/symbols_linux.c @@ -168,6 +168,8 @@ struct int_symbol platform_symbols_table[] = { #endif { IPV6_TCLASS, "IPV6_TCLASS" }, { IPV6_HOPLIMIT, "IPV6_HOPLIMIT" }, + { IP_BIND_ADDRESS_NO_PORT, "IP_BIND_ADDRESS_NO_PORT" }, + { IP_LOCAL_PORT_RANGE, "IP_LOCAL_PORT_RANGE" }, { TCP_NODELAY, "TCP_NODELAY" }, { TCP_MAXSEG, "TCP_MAXSEG" }, diff --git a/gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_fail_after_500usec.pkt b/gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_fail_after_500usec.pkt new file mode 100644 index 00000000..066ffce5 --- /dev/null +++ b/gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_fail_after_500usec.pkt @@ -0,0 +1,51 @@ +// Check that TIME-WAIT 4-tuple can't be reused after less than a millisecond + +--tcp_ts_tick_usecs=1000 +--tolerance_percent=1 + +` +../../common/defaults.sh +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1 # msec +` + +// +// Prime TIME-WAIT - open first connection and close it. +// Use source port 0xdead (57005). +// + + 0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + + +0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 + +// +// Reincarnation after less than 1 msec - open and close second connection. +// Use the same source port. +// + + +0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + +// Delay for 500 usec and assume it takes packetdrill less than 500 usec to +// reincarnate a connection (the interval from last ACK to next connect()), so +// the total delay is less than 1 msec. On a consumer laptop connection +// reincarnation takes ~200 usec. ++.0005 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address) + + +0 close(3) = 0 + diff --git a/gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_ok_after_1msec.pkt b/gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_ok_after_1msec.pkt new file mode 100644 index 00000000..241cf932 --- /dev/null +++ b/gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_ok_after_1msec.pkt @@ -0,0 +1,60 @@ +// Check that TIME-WAIT 4-tuple can be reused after a 1 millisecond period + +--tcp_ts_tick_usecs=1000 +--tolerance_percent=1 + +` +../../common/defaults.sh +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1 # msec +` + +// +// Prime TIME-WAIT - open first connection and close it. +// Use source port 0xdead (57005). +// + + 0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + + +0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 <...> + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 + +// +// Reincarnation after 1 millisecond - open and close second connection. +// Use the same source port. +// + + +0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + + +// Delay for 1 millisecond, plus an extra millisecond to avoid test flakiness. +// Kernel TIME-WAIT reuse timer has millisecond resolution. Actual reuse delay +// can be up to 1 msec longer than the setting due to timestamp rounding. ++.002 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + +// NOTE: Packetdrill doesn't actually track timestamps across connection +// reincarnations despite the fact that the random offset doesn't change. + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 diff --git a/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_0sec.pkt b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_0sec.pkt new file mode 100644 index 00000000..79f1c15e --- /dev/null +++ b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_0sec.pkt @@ -0,0 +1,46 @@ +// Check that TIME-WAIT 4-tuple can't be reused immediately + +--tcp_ts_tick_usecs=1000 +--tolerance_percent=1 + +` +../../common/defaults.sh +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec +` + +// +// Prime TIME-WAIT - open first connection and close it. +// Use source port 0xdead (57005). +// + + 0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + + +0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 + +// +// Reincarnation without delay - open and close second connection. +// Use the same source port. +// + + +0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + ++0.00 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address) + + +0 close(3) = 0 diff --git a/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_1sec_without_tsopt.pkt b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_1sec_without_tsopt.pkt new file mode 100644 index 00000000..00b45bdf --- /dev/null +++ b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_1sec_without_tsopt.pkt @@ -0,0 +1,47 @@ +// Check that TIME-WAIT 4-tuple can't be reused after the TIME-WAIT reuse delay +// when peer doesn't support TSopt + +--tcp_ts_tick_usecs=1000 +--tolerance_percent=1 + +` +../../common/defaults.sh +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec +` + +// +// Prime TIME-WAIT - open first connection and close it. +// Use source port 0xdead (57005). +// + + 0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + + +0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 + +// +// Reincarnation after 1 second - open and close second connection. +// Use same source port. +// + + +0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + ++1.00 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address) + + +0 close(3) = 0 diff --git a/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_990msec.pkt b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_990msec.pkt new file mode 100644 index 00000000..726d33b0 --- /dev/null +++ b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_990msec.pkt @@ -0,0 +1,47 @@ +// Check that TCP 4-tuple can't be reincarnated when we are close to TIME-WAIT +// reuse threshold (990 msec have elapsed). + +--tcp_ts_tick_usecs=1000 +--tolerance_percent=1 + +` +../../common/defaults.sh +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec +` + +// +// Prime TIME-WAIT - open first connection and close it. +// Use source port 0xdead (57005). +// + + 0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + + +0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 + +// +// Reincarnation after almost 1 second - open and close second connection. +// Use the same source port. +// + + +0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + ++0.99 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address) + + +0 close(3) = 0 diff --git a/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_0sec_with_portlock.pkt b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_0sec_with_portlock.pkt new file mode 100644 index 00000000..8f234aba --- /dev/null +++ b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_0sec_with_portlock.pkt @@ -0,0 +1,58 @@ +// Check that TIME-WAIT 4-tuple can be reused immediately if bind() was used to +// reserve the source port. + +--bind_port=57005 +--tcp_ts_tick_usecs=1000 +--tolerance_percent=1 + +` +../../common/defaults.sh +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec +` + +// +// Prime TIME-WAIT - open first connection and close it. +// Use source port 57005. Reserve it with bind(). +// + + 0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 + +0 bind(3, ..., ...) = 0 + + +0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 + +// +// Reincarnation without delay - open and close second connection. +// Use same source port. +// + + +0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 + +0 bind(3, ..., ...) = 0 + ++0.00 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + +// NOTE: Packetdrill doesn't actually track timestamps across connection +// reincarnations despite the fact that the random offset doesn't change. + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 diff --git a/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_1sec.pkt b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_1sec.pkt new file mode 100644 index 00000000..4fd5b404 --- /dev/null +++ b/gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_1sec.pkt @@ -0,0 +1,59 @@ +// Check that TIME-WAIT 4-tuple can be reused after a 1 second period. + +--tcp_ts_tick_usecs=1000 +--tolerance_percent=1 + +` +../../common/defaults.sh +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable +../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec +` + +// +// Prime TIME-WAIT - open first connection and close it. +// Use source port 0xdead (57005). +// + + 0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + + +0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 <...> + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2 + +// +// Reincarnation after 1 second - open and close second connection. +// Use the same source port. +// + + +0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3 + + +0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0 + +0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0 + +// Avoid spurious failures with an extra 1 millisecond delay. +// Kernel TIME-WAIT reuse timer has millisecond resolution. Actual reuse delay +// can be up to 1 msec longer than the setting due to timestamp rounding. ++1.01 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress) + +// NOTE: Packetdrill doesn't actually track timestamps across connection +// reincarnations despite the fact that the random offset doesn't change. + +0 > S 0:0(0) ++.001 < S. 0:0(0) ack 1 win 65535 + +0 > . 1:1(0) ack 1 + + +0 close(3) = 0 + + +0 > F. 1:1(0) ack 1 ++.001 < F. 1:1(0) ack 2 win 8 + +0 > . 2:2(0) ack 2