BIG WireGuard TCP implementation patch Kernel baseline: Linux v6.8 (90d1f30371ae3337beb01666b226320728d35c70) Tools baseline: wireguard-tools a998407747005ea7e4e0258d96f105c97241e1d3 Target repository: https://github.com/secwest/WireguardTCP.git Target source commit: ae38aea0ac0a0d08c7410500ff020a414d13a627 This patch contains only the allowlisted kernel module, UAPI, and wg source changes needed to build and run WireguardTCP. Upstream kernel crypto, SIMD, and architecture assembly are not modified by this patch. Prepare the baseline from clean official checkouts: git clone https://github.com/torvalds/linux.git linux git -C linux checkout 90d1f30371ae3337beb01666b226320728d35c70 git clone https://github.com/WireGuard/wireguard-tools.git wireguard-tools git -C wireguard-tools checkout a998407747005ea7e4e0258d96f105c97241e1d3 mkdir WireguardTCP-patch-base cp -a linux/drivers/net/wireguard WireguardTCP-patch-base/kernel mkdir -p WireguardTCP-patch-base/include/uapi/linux cp linux/include/uapi/linux/wireguard.h WireguardTCP-patch-base/include/uapi/linux/ cp -a wireguard-tools/src WireguardTCP-patch-base/tools Apply and build on Ubuntu 24.04 with matching kernel headers installed: cd WireguardTCP-patch-base git apply --binary /path/to/BIG-WireguardTCP-Patch make -C tools -j$(nproc) make -C /lib/modules/$(uname -r)/build M=$PWD/kernel CONFIG_WIREGUARD=m -j$(nproc) modules diff --git a/include/uapi/linux/wireguard.h b/include/uapi/linux/wireguard.h index 0efd52c3687d981dcac4b074cc0aea3362457e9d..d6d288022773b1a8697345cb304058441e1c9984 100644 --- a/include/uapi/linux/wireguard.h +++ b/include/uapi/linux/wireguard.h @@ -29,6 +29,7 @@ * WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN * WGDEVICE_A_LISTEN_PORT: NLA_U16 * WGDEVICE_A_FWMARK: NLA_U32 + * WGDEVICE_A_TRANSPORT: NLA_U8, WG_TRANSPORT_UDP or WG_TRANSPORT_TCP * WGDEVICE_A_PEERS: NLA_NESTED * 0: NLA_NESTED * WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN @@ -81,8 +82,11 @@ * WGDEVICE_A_FLAGS: NLA_U32, 0 or WGDEVICE_F_REPLACE_PEERS if all current * peers should be removed prior to adding the list below. * WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove - * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly + * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly at interface-up; + * changing it on a running TCP device returns EBUSY * WGDEVICE_A_FWMARK: NLA_U32, 0 to disable + * WGDEVICE_A_TRANSPORT: NLA_U8, WG_TRANSPORT_UDP or WG_TRANSPORT_TCP; + * omission preserves the current transport * WGDEVICE_A_PEERS: NLA_NESTED * 0: NLA_NESTED * WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN @@ -101,6 +105,10 @@ * WGALLOWEDIP_A_FAMILY: NLA_U16 * WGALLOWEDIP_A_IPADDR: struct in_addr or struct in6_addr * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * WGALLOWEDIP_A_FLAGS: NLA_U32, WGALLOWEDIP_F_REMOVE_ME if + * the specified IP should be removed; + * otherwise, this IP will be added if + * it is not already present. * 0: NLA_NESTED * ... * 0: NLA_NESTED @@ -136,6 +144,9 @@ #define WG_KEY_LEN 32 +#define WG_TRANSPORT_UDP 0 +#define WG_TRANSPORT_TCP 1 + enum wg_cmd { WG_CMD_GET_DEVICE, WG_CMD_SET_DEVICE, @@ -157,6 +168,7 @@ enum wgdevice_attribute { WGDEVICE_A_LISTEN_PORT, WGDEVICE_A_FWMARK, WGDEVICE_A_PEERS, + WGDEVICE_A_TRANSPORT, __WGDEVICE_A_LAST }; #define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1) @@ -184,11 +196,16 @@ enum wgpeer_attribute { }; #define WGPEER_A_MAX (__WGPEER_A_LAST - 1) +enum wgallowedip_flag { + WGALLOWEDIP_F_REMOVE_ME = 1U << 0, + __WGALLOWEDIP_F_ALL = WGALLOWEDIP_F_REMOVE_ME +}; enum wgallowedip_attribute { WGALLOWEDIP_A_UNSPEC, WGALLOWEDIP_A_FAMILY, WGALLOWEDIP_A_IPADDR, WGALLOWEDIP_A_CIDR_MASK, + WGALLOWEDIP_A_FLAGS, __WGALLOWEDIP_A_LAST }; #define WGALLOWEDIP_A_MAX (__WGALLOWEDIP_A_LAST - 1) diff --git a/kernel/Makefile b/kernel/Makefile index dbe1f8514efc3df1175762c62d71f88fe6327835..36bfc716d4d57c58121361a63bf29611a88e603d 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -1,5 +1,14 @@ ccflags-y := -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt' +ccflags-y += -I$(src)/../include -I$(src)/../include/uapi +ccflags-y += -include $(src)/wireguard_tcp_uapi.h ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DDEBUG + +# WireGuard TCP debug levels (add to EXTRA_CFLAGS or uncomment here): +# -DWG_TCP_VERBOSE Function enter/exit traces, param dumps (very noisy) +# -DWG_TCP_DIAG TCP performance diagnostics (cwnd, rtt, retrans) +# Neither No debug output (production) +# Example: make ... EXTRA_CFLAGS="-DWG_TCP_DIAG" +# ccflags-y += -DWG_TCP_VERBOSE -DWG_TCP_DIAG wireguard-y := main.o wireguard-y += noise.o wireguard-y += device.o @@ -9,6 +18,8 @@ wireguard-y += queueing.o wireguard-y += send.o wireguard-y += receive.o wireguard-y += socket.o +wireguard-y += wg_tcp.o +wireguard-y += wg_tcp_debug.o wireguard-y += peerlookup.o wireguard-y += allowedips.o wireguard-y += ratelimiter.o diff --git a/kernel/allowedips.c b/kernel/allowedips.c index 0ba714ca5185cd94124bd121a49117b79b9898bc..411a940b0e548fe484bfcaa10e14206a1c7d23b3 100644 --- a/kernel/allowedips.c +++ b/kernel/allowedips.c @@ -5,6 +5,7 @@ #include "allowedips.h" #include "peer.h" +#include "wg_tcp_debug.h" enum { MAX_ALLOWEDIPS_DEPTH = 129 }; @@ -12,17 +13,20 @@ static struct kmem_cache *node_cache; static void swap_endian(u8 *dst, const u8 *src, u8 bits) { + wg_dbg("Entering swap_endian(dst=%px, src=%px, bits=%u)\n", dst, src, bits); if (bits == 32) { *(u32 *)dst = be32_to_cpu(*(const __be32 *)src); } else if (bits == 128) { - ((u64 *)dst)[0] = be64_to_cpu(((const __be64 *)src)[0]); - ((u64 *)dst)[1] = be64_to_cpu(((const __be64 *)src)[1]); + ((u64 *)dst)[0] = get_unaligned_be64(src); + ((u64 *)dst)[1] = get_unaligned_be64(src + 8); } + wg_dbg("Exiting swap_endian\n"); } static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src, u8 cidr, u8 bits) { + wg_dbg("Entering copy_and_assign_cidr(node=%px, src=%px, cidr=%u, bits=%u)\n", node, src, cidr, bits); node->cidr = cidr; node->bit_at_a = cidr / 8U; #ifdef __LITTLE_ENDIAN @@ -31,30 +35,39 @@ static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src, node->bit_at_b = 7U - (cidr % 8U); node->bitlen = bits; memcpy(node->bits, src, bits / 8U); + wg_dbg("Exiting copy_and_assign_cidr\n"); } static inline u8 choose(struct allowedips_node *node, const u8 *key) { - return (key[node->bit_at_a] >> node->bit_at_b) & 1; + wg_dbg("Entering choose(node=%px, key=%px)\n", node, key); + u8 result = (key[node->bit_at_a] >> node->bit_at_b) & 1; + wg_dbg("Exiting choose with result=%u\n", result); + return result; } static void push_rcu(struct allowedips_node **stack, struct allowedips_node __rcu *p, unsigned int *len) { + wg_dbg("Entering push_rcu(stack=%px, p=%px, len=%u)\n", stack, p, *len); if (rcu_access_pointer(p)) { if (WARN_ON(IS_ENABLED(DEBUG) && *len >= MAX_ALLOWEDIPS_DEPTH)) return; stack[(*len)++] = rcu_dereference_raw(p); } + wg_dbg("Exiting push_rcu\n"); } static void node_free_rcu(struct rcu_head *rcu) { + wg_dbg("Entering node_free_rcu(rcu=%px)\n", rcu); kmem_cache_free(node_cache, container_of(rcu, struct allowedips_node, rcu)); + wg_dbg("Exiting node_free_rcu\n"); } static void root_free_rcu(struct rcu_head *rcu) { + wg_dbg("Entering root_free_rcu(rcu=%px)\n", rcu); struct allowedips_node *node, *stack[MAX_ALLOWEDIPS_DEPTH] = { container_of(rcu, struct allowedips_node, rcu) }; unsigned int len = 1; @@ -64,10 +77,12 @@ static void root_free_rcu(struct rcu_head *rcu) push_rcu(stack, node->bit[1], &len); kmem_cache_free(node_cache, node); } + wg_dbg("Exiting root_free_rcu\n"); } static void root_remove_peer_lists(struct allowedips_node *root) { + wg_dbg("Entering root_remove_peer_lists(root=%px)\n", root); struct allowedips_node *node, *stack[MAX_ALLOWEDIPS_DEPTH] = { root }; unsigned int len = 1; @@ -77,40 +92,53 @@ static void root_remove_peer_lists(struct allowedips_node *root) if (rcu_access_pointer(node->peer)) list_del(&node->peer_list); } + wg_dbg("Exiting root_remove_peer_lists\n"); } static unsigned int fls128(u64 a, u64 b) { - return a ? fls64(a) + 64U : fls64(b); + wg_dbg("Entering fls128(a=%llu, b=%llu)\n", a, b); + unsigned int result = a ? fls64(a) + 64U : fls64(b); + wg_dbg("Exiting fls128 with result=%u\n", result); + return result; } static u8 common_bits(const struct allowedips_node *node, const u8 *key, u8 bits) { + wg_dbg("Entering common_bits(node=%px, key=%px, bits=%u)\n", node, key, bits); + u8 result; if (bits == 32) - return 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key); + result = 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key); else if (bits == 128) - return 128U - fls128( + result = 128U - fls128( *(const u64 *)&node->bits[0] ^ *(const u64 *)&key[0], *(const u64 *)&node->bits[8] ^ *(const u64 *)&key[8]); - return 0; + else + result = 0; + wg_dbg("Exiting common_bits with result=%u\n", result); + return result; } static bool prefix_matches(const struct allowedips_node *node, const u8 *key, u8 bits) { + wg_dbg("Entering prefix_matches(node=%px, key=%p, bits=%u)\n", node, key, bits); /* This could be much faster if it actually just compared the common * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and * the rest, but it turns out that common_bits is already super fast on * modern processors, even taking into account the unfortunate bswap. * So, we just inline it like this instead. */ - return common_bits(node, key, bits) >= node->cidr; + bool result = common_bits(node, key, bits) >= node->cidr; + wg_dbg("Exiting prefix_matches with result=%d\n", result); + return result; } static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits, const u8 *key) { + wg_dbg("Entering find_node(trie=%px, bits=%u, key=%px)\n", trie, bits, key); struct allowedips_node *node = trie, *found = NULL; while (node && prefix_matches(node, key, bits)) { @@ -120,13 +148,16 @@ static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits, break; node = rcu_dereference_bh(node->bit[choose(node, key)]); } + wg_dbg("Exiting find_node with found=%px\n", found); return found; } + /* Returns a strong reference to a peer */ static struct wg_peer *lookup(struct allowedips_node __rcu *root, u8 bits, const void *be_ip) { + wg_dbg("Entering lookup(root=%px, bits=%u, be_ip=%px)\n", root, bits, be_ip); /* Aligned so it can be passed to fls/fls64 */ u8 ip[16] __aligned(__alignof(u64)); struct allowedips_node *node; @@ -143,13 +174,16 @@ retry: goto retry; } rcu_read_unlock_bh(); + wg_dbg("Exiting lookup with peer=%px\n", peer); return peer; } + static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key, u8 cidr, u8 bits, struct allowedips_node **rnode, struct mutex *lock) { + wg_dbg("Entering node_placement(trie=%px, key=%px, cidr=%u, bits=%u, rnode=%px, lock=%px)\n", trie, key, cidr, bits, rnode, lock); struct allowedips_node *node = rcu_dereference_protected(trie, lockdep_is_held(lock)); struct allowedips_node *parent = NULL; bool exact = false; @@ -163,24 +197,30 @@ static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key, node = rcu_dereference_protected(parent->bit[choose(parent, key)], lockdep_is_held(lock)); } *rnode = parent; + wg_dbg("Exiting node_placement with exact=%d\n", exact); return exact; } static inline void connect_node(struct allowedips_node __rcu **parent, u8 bit, struct allowedips_node *node) { + wg_dbg("Entering connect_node(parent=%px, bit=%u, node=%px)\n", parent, bit, node); node->parent_bit_packed = (unsigned long)parent | bit; rcu_assign_pointer(*parent, node); + wg_dbg("Exiting connect_node\n"); } static inline void choose_and_connect_node(struct allowedips_node *parent, struct allowedips_node *node) { + wg_dbg("Entering choose_and_connect_node(parent=%px, node=%px)\n", parent, node); u8 bit = choose(parent, node->bits); connect_node(&parent->bit[bit], bit, node); + wg_dbg("Exiting choose_and_connect_node\n"); } static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, u8 cidr, struct wg_peer *peer, struct mutex *lock) { + wg_dbg("Entering add(trie=%px, bits=%u, key=%px, cidr=%u, peer=%px, lock=%px)\n", trie, bits, key, cidr, peer, lock); struct allowedips_node *node, *parent, *down, *newnode; if (unlikely(cidr > bits || !peer)) @@ -194,11 +234,13 @@ static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, list_add_tail(&node->peer_list, &peer->allowedips_list); copy_and_assign_cidr(node, key, cidr, bits); connect_node(trie, 2, node); + wg_dbg("Exiting add with return 0\n"); return 0; } if (node_placement(*trie, key, cidr, bits, &node, lock)) { rcu_assign_pointer(node->peer, peer); list_move_tail(&node->peer_list, &peer->allowedips_list); + wg_dbg("Exiting add with return 0\n"); return 0; } @@ -216,6 +258,7 @@ static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, down = rcu_dereference_protected(node->bit[bit], lockdep_is_held(lock)); if (!down) { connect_node(&node->bit[bit], bit, newnode); + wg_dbg("Exiting add with return 0\n"); return 0; } } @@ -228,6 +271,7 @@ static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, connect_node(trie, 2, newnode); else choose_and_connect_node(parent, newnode); + wg_dbg("Exiting add with return 0\n"); return 0; } @@ -246,17 +290,72 @@ static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, connect_node(trie, 2, node); else choose_and_connect_node(parent, node); + wg_dbg("Exiting add with return 0\n"); + return 0; +} + +static void remove_node(struct allowedips_node *node, struct mutex *lock) +{ + struct allowedips_node *child, **parent_bit, *parent; + bool free_parent; + + list_del_init(&node->peer_list); + RCU_INIT_POINTER(node->peer, NULL); + if (node->bit[0] && node->bit[1]) + return; + child = rcu_dereference_protected( + node->bit[!rcu_access_pointer(node->bit[0])], + lockdep_is_held(lock)); + if (child) + child->parent_bit_packed = node->parent_bit_packed; + parent_bit = (struct allowedips_node **)(node->parent_bit_packed & ~3UL); + *parent_bit = child; + parent = (void *)parent_bit - + offsetof(struct allowedips_node, bit[node->parent_bit_packed & 1]); + free_parent = !rcu_access_pointer(node->bit[0]) && + !rcu_access_pointer(node->bit[1]) && + (node->parent_bit_packed & 3) <= 1 && + !rcu_access_pointer(parent->peer); + if (free_parent) + child = rcu_dereference_protected( + parent->bit[!(node->parent_bit_packed & 1)], + lockdep_is_held(lock)); + call_rcu(&node->rcu, node_free_rcu); + if (!free_parent) + return; + if (child) + child->parent_bit_packed = parent->parent_bit_packed; + *(struct allowedips_node **)(parent->parent_bit_packed & ~3UL) = child; + call_rcu(&parent->rcu, node_free_rcu); +} + +static int remove(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + struct allowedips_node *node; + + if (unlikely(cidr > bits)) + return -EINVAL; + if (!rcu_access_pointer(*trie) || + !node_placement(*trie, key, cidr, bits, &node, lock) || + peer != rcu_access_pointer(node->peer)) + return 0; + + remove_node(node, lock); return 0; } void wg_allowedips_init(struct allowedips *table) { + wg_dbg("Entering wg_allowedips_init(table=%px)\n", table); table->root4 = table->root6 = NULL; table->seq = 1; + wg_dbg("Exiting wg_allowedips_init\n"); } void wg_allowedips_free(struct allowedips *table, struct mutex *lock) { + wg_dbg("Entering wg_allowedips_free(table=%px, lock=%px)\n", table, lock); struct allowedips_node __rcu *old4 = table->root4, *old6 = table->root6; ++table->seq; @@ -276,72 +375,74 @@ void wg_allowedips_free(struct allowedips *table, struct mutex *lock) root_remove_peer_lists(node); call_rcu(&node->rcu, root_free_rcu); } + wg_dbg("Exiting wg_allowedips_free\n"); } int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, u8 cidr, struct wg_peer *peer, struct mutex *lock) { + wg_dbg("Entering wg_allowedips_insert_v4(table=%px, ip=%px, cidr=%u, peer=%px, lock=%px)\n", table, ip, cidr, peer, lock); /* Aligned so it can be passed to fls */ u8 key[4] __aligned(__alignof(u32)); ++table->seq; swap_endian(key, (const u8 *)ip, 32); - return add(&table->root4, 32, key, cidr, peer, lock); + int result = add(&table->root4, 32, key, cidr, peer, lock); + wg_dbg("Exiting wg_allowedips_insert_v4 with result=%d\n", result); + return result; } int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, u8 cidr, struct wg_peer *peer, struct mutex *lock) { + wg_dbg("Entering wg_allowedips_insert_v6(table=%px, ip=%px, cidr=%u, peer=%px, lock=%px)\n", table, ip, cidr, peer, lock); /* Aligned so it can be passed to fls64 */ u8 key[16] __aligned(__alignof(u64)); ++table->seq; swap_endian(key, (const u8 *)ip, 128); - return add(&table->root6, 128, key, cidr, peer, lock); + int result = add(&table->root6, 128, key, cidr, peer, lock); + wg_dbg("Exiting wg_allowedips_insert_v6 with result=%d\n", result); + return result; +} + +int wg_allowedips_remove_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + u8 key[4] __aligned(__alignof(u32)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 32); + return remove(&table->root4, 32, key, cidr, peer, lock); +} + +int wg_allowedips_remove_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + u8 key[16] __aligned(__alignof(u64)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 128); + return remove(&table->root6, 128, key, cidr, peer, lock); } void wg_allowedips_remove_by_peer(struct allowedips *table, struct wg_peer *peer, struct mutex *lock) { - struct allowedips_node *node, *child, **parent_bit, *parent, *tmp; - bool free_parent; + wg_dbg("Entering wg_allowedips_remove_by_peer(table=%px, peer=%px, lock=%px)\n", table, peer, lock); + struct allowedips_node *node, *tmp; if (list_empty(&peer->allowedips_list)) return; ++table->seq; - list_for_each_entry_safe(node, tmp, &peer->allowedips_list, peer_list) { - list_del_init(&node->peer_list); - RCU_INIT_POINTER(node->peer, NULL); - if (node->bit[0] && node->bit[1]) - continue; - child = rcu_dereference_protected(node->bit[!rcu_access_pointer(node->bit[0])], - lockdep_is_held(lock)); - if (child) - child->parent_bit_packed = node->parent_bit_packed; - parent_bit = (struct allowedips_node **)(node->parent_bit_packed & ~3UL); - *parent_bit = child; - parent = (void *)parent_bit - - offsetof(struct allowedips_node, bit[node->parent_bit_packed & 1]); - free_parent = !rcu_access_pointer(node->bit[0]) && - !rcu_access_pointer(node->bit[1]) && - (node->parent_bit_packed & 3) <= 1 && - !rcu_access_pointer(parent->peer); - if (free_parent) - child = rcu_dereference_protected( - parent->bit[!(node->parent_bit_packed & 1)], - lockdep_is_held(lock)); - call_rcu(&node->rcu, node_free_rcu); - if (!free_parent) - continue; - if (child) - child->parent_bit_packed = parent->parent_bit_packed; - *(struct allowedips_node **)(parent->parent_bit_packed & ~3UL) = child; - call_rcu(&parent->rcu, node_free_rcu); - } + list_for_each_entry_safe(node, tmp, &peer->allowedips_list, peer_list) + remove_node(node, lock); + wg_dbg("Exiting wg_allowedips_remove_by_peer\n"); } int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr) { + wg_dbg("Entering wg_allowedips_read_node(node=%px, ip=%px, cidr=%px)\n", node, ip, cidr); const unsigned int cidr_bytes = DIV_ROUND_UP(node->cidr, 8U); swap_endian(ip, node->bits, node->bitlen); memset(ip + cidr_bytes, 0, node->bitlen / 8U - cidr_bytes); @@ -349,41 +450,58 @@ int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr) ip[cidr_bytes - 1U] &= ~0U << (-node->cidr % 8U); *cidr = node->cidr; - return node->bitlen == 32 ? AF_INET : AF_INET6; + int result = node->bitlen == 32 ? AF_INET : AF_INET6; + wg_dbg("Exiting wg_allowedips_read_node with result=%d\n", result); + return result; } /* Returns a strong reference to a peer */ struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, struct sk_buff *skb) { + wg_dbg("Entering wg_allowedips_lookup_dst(table=%px, skb=%px)\n", table, skb); + struct wg_peer *result; if (skb->protocol == htons(ETH_P_IP)) - return lookup(table->root4, 32, &ip_hdr(skb)->daddr); + result = lookup(table->root4, 32, &ip_hdr(skb)->daddr); else if (skb->protocol == htons(ETH_P_IPV6)) - return lookup(table->root6, 128, &ipv6_hdr(skb)->daddr); - return NULL; + result = lookup(table->root6, 128, &ipv6_hdr(skb)->daddr); + else + result = NULL; + wg_dbg("Exiting wg_allowedips_lookup_dst with result=%px\n", result); + return result; } /* Returns a strong reference to a peer */ struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, struct sk_buff *skb) { + wg_dbg("Entering wg_allowedips_lookup_src(table=%px, skb=%px)\n", table, skb); + struct wg_peer *result; if (skb->protocol == htons(ETH_P_IP)) - return lookup(table->root4, 32, &ip_hdr(skb)->saddr); + result = lookup(table->root4, 32, &ip_hdr(skb)->saddr); else if (skb->protocol == htons(ETH_P_IPV6)) - return lookup(table->root6, 128, &ipv6_hdr(skb)->saddr); - return NULL; + result = lookup(table->root6, 128, &ipv6_hdr(skb)->saddr); + else + result = NULL; + wg_dbg("Exiting wg_allowedips_lookup_src with result=%px\n", result); + return result; } int __init wg_allowedips_slab_init(void) { + wg_dbg("Entering wg_allowedips_slab_init()\n"); node_cache = KMEM_CACHE(allowedips_node, 0); - return node_cache ? 0 : -ENOMEM; + int result = node_cache ? 0 : -ENOMEM; + wg_dbg("Exiting wg_allowedips_slab_init with result=%d\n", result); + return result; } void wg_allowedips_slab_uninit(void) { + wg_dbg("Entering wg_allowedips_slab_uninit()\n"); rcu_barrier(); kmem_cache_destroy(node_cache); + wg_dbg("Exiting wg_allowedips_slab_uninit\n"); } #include "selftest/allowedips.c" diff --git a/kernel/allowedips.h b/kernel/allowedips.h index 2346c797eb4d877d76504e3e23d4d2b5dc831f1a..931958cb6e100c99c71d66e03e2ba4ffb199f07d 100644 --- a/kernel/allowedips.h +++ b/kernel/allowedips.h @@ -38,6 +38,10 @@ int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, u8 cidr, struct wg_peer *peer, struct mutex *lock); int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, u8 cidr, struct wg_peer *peer, struct mutex *lock); +int wg_allowedips_remove_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +int wg_allowedips_remove_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); void wg_allowedips_remove_by_peer(struct allowedips *table, struct wg_peer *peer, struct mutex *lock); /* The ip input pointer should be __aligned(__alignof(u64))) */ diff --git a/kernel/cookie.h b/kernel/cookie.h index c4bd61ca03f24230fd0f0cdaae231cb7a6aa29b8..10ac1f5ac65d142d37f1fcdb3bcb7276077f9168 100644 --- a/kernel/cookie.h +++ b/kernel/cookie.h @@ -38,6 +38,24 @@ enum cookie_mac_state { VALID_MAC_WITH_COOKIE }; +enum cookie_validation_action { + WG_COOKIE_DROP, + WG_COOKIE_ACCEPT, + WG_COOKIE_CHALLENGE +}; + +static inline enum cookie_validation_action +wg_cookie_validation_action(bool under_load, enum cookie_mac_state mac_state) +{ + if (under_load && mac_state == VALID_MAC_WITH_COOKIE) + return WG_COOKIE_ACCEPT; + if (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) + return WG_COOKIE_ACCEPT; + if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) + return WG_COOKIE_CHALLENGE; + return WG_COOKIE_DROP; +} + void wg_cookie_checker_init(struct cookie_checker *checker, struct wg_device *wg); void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker); @@ -56,4 +74,8 @@ void wg_cookie_message_create(struct message_handshake_cookie *src, void wg_cookie_message_consume(struct message_handshake_cookie *src, struct wg_device *wg); +#ifdef DEBUG +bool wg_cookie_policy_selftest(void); +#endif + #endif /* _WG_COOKIE_H */ diff --git a/kernel/device.c b/kernel/device.c index deb9636b0ecf8f47e832a0b07e9e049ba19bdf16..2c55e2c39da988231bd643372d961dea77b0cddb 100644 --- a/kernel/device.c +++ b/kernel/device.c @@ -1,10 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ #include "queueing.h" #include "socket.h" +#include "wg_tcp.h" #include "timers.h" #include "device.h" #include "ratelimiter.h" @@ -19,14 +21,27 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include +#include +#include "wg_tcp_debug.h" static LIST_HEAD(device_list); +static unsigned int wg_net_id; + +struct wg_net { + struct net *net; + struct notifier_block fib_notifier; + struct delayed_work fib_dispatch_work; + bool fib_registered; +}; static int wg_open(struct net_device *dev) { @@ -34,7 +49,12 @@ static int wg_open(struct net_device *dev) struct inet6_dev *dev_v6 = __in6_dev_get(dev); struct wg_device *wg = netdev_priv(dev); struct wg_peer *peer; - int ret; + u16 requested_port = wg->incoming_port; + int ret = 0; + + wg_dbg("Entering wg_open: dev=%px\n", dev); + WRITE_ONCE(wg->tcp_cleanup_scheduled, + wg->transport == WG_TRANSPORT_TCP); if (dev_v4) { /* At some point we might put this check near the ip_rt_send_ @@ -47,17 +67,68 @@ static int wg_open(struct net_device *dev) if (dev_v6) dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; - mutex_lock(&wg->device_update_lock); + + wg->listener_active = false; + /* Bind UDP first so port zero retains WireGuard's random-port semantics. + * TCP then uses the concrete port selected by the companion UDP socket. + */ ret = wg_socket_init(wg, wg->incoming_port); - if (ret < 0) - goto out; + if (ret < 0) { + WRITE_ONCE(wg->tcp_cleanup_scheduled, false); + return ret; + } + if (wg->transport == WG_TRANSPORT_TCP) { + if (!wg->tcp_auth_wq) { + wg->tcp_auth_wq = alloc_workqueue("wg-tcp-auth-%s", + WQ_UNBOUND | WQ_MEM_RECLAIM, + 0, dev->name); + if (!wg->tcp_auth_wq) { + ret = -ENOMEM; + goto err_tcp_open; + } + } + ret = wg_tcp_listener_socket_init(wg, wg->incoming_port); + if (ret < 0) + goto err_tcp_open; + } + mutex_lock(&wg->device_update_lock); list_for_each_entry(peer, &wg->peer_list, peer_list) { + bool queue_tcp_retry = false; + bool tcp_quarantined = false; + + if (wg->transport == WG_TRANSPORT_TCP) { + spin_lock_bh(&peer->tcp_lock); + tcp_quarantined = peer->tcp_teardown_quarantined; + if (!tcp_quarantined) + peer->tcp_stopping = false; + if (!tcp_quarantined && peer->peer_endpoint_set && + !peer->tcp_retry_scheduled && + !peer->tcp_outbound_remove_scheduled) { + peer->tcp_retry_scheduled = true; + queue_tcp_retry = true; + } + if (queue_tcp_retry) + mod_delayed_work(system_wq, &peer->tcp_retry_work, 0); + spin_unlock_bh(&peer->tcp_lock); + } + if (tcp_quarantined) + continue; wg_packet_send_staged_packets(peer); if (peer->persistent_keepalive_interval) wg_packet_send_keepalive(peer); } -out: mutex_unlock(&wg->device_update_lock); + wg_dbg("Exiting wg_open: dev=%px, ret=%d\n", dev, ret); + return ret; + +err_tcp_open: + WRITE_ONCE(wg->tcp_cleanup_scheduled, false); + wg_tcp_listener_socket_release(wg); + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + wg_destruct_tcp_connection_list(wg); + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + wg_socket_reinit(wg, NULL, NULL); + wg->incoming_port = requested_port; return ret; } @@ -66,16 +137,22 @@ static int wg_pm_notification(struct notifier_block *nb, unsigned long action, v struct wg_device *wg; struct wg_peer *peer; + wg_dbg("Entering wg_pm_notification: nb=%px, action=%lu, data=%px\n", nb, action, data); + /* If the machine is constantly suspending and resuming, as part of * its normal operation rather than as a somewhat rare event, then we * don't actually want to clear keys. */ if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || - IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP)) + IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP)) { + wg_dbg("Exiting wg_pm_notification (no action): nb=%px, action=%lu, data=%px\n", nb, action, data); return 0; + } - if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) + if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) { + wg_dbg("Exiting wg_pm_notification (no action): nb=%px, action=%lu, data=%px\n", nb, action, data); return 0; + } rtnl_lock(); list_for_each_entry(wg, &device_list, device_list) { @@ -89,6 +166,7 @@ static int wg_pm_notification(struct notifier_block *nb, unsigned long action, v } rtnl_unlock(); rcu_barrier(); + wg_dbg("Exiting wg_pm_notification: nb=%px, action=%lu, data=%px\n", nb, action, data); return 0; } @@ -99,6 +177,8 @@ static int wg_vm_notification(struct notifier_block *nb, unsigned long action, v struct wg_device *wg; struct wg_peer *peer; + wg_dbg("Entering wg_vm_notification: nb=%px, action=%lu, data=%px\n", nb, action, data); + rtnl_lock(); list_for_each_entry(wg, &device_list, device_list) { mutex_lock(&wg->device_update_lock); @@ -107,18 +187,175 @@ static int wg_vm_notification(struct notifier_block *nb, unsigned long action, v mutex_unlock(&wg->device_update_lock); } rtnl_unlock(); + wg_dbg("Exiting wg_vm_notification: nb=%px, action=%lu, data=%px\n", nb, action, data); return 0; } static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification }; +static void wg_tcp_route_change_worker(struct work_struct *work) +{ + struct wg_device *wg = container_of(work, struct wg_device, + tcp_route_work.work); + struct wg_peer *peer; + + mutex_lock(&wg->device_update_lock); + if (wg->transport == WG_TRANSPORT_TCP && + READ_ONCE(wg->tcp_cleanup_scheduled) && netif_running(wg->dev) && + rcu_access_pointer(wg->creating_net)) { + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_socket_clear_peer_endpoint_src(peer); + wg_tcp_peer_request_reconnect(peer); + } + } + mutex_unlock(&wg->device_update_lock); +} + +static void wg_tcp_fib_dispatch_worker(struct work_struct *work) +{ + struct wg_net *wn = container_of(work, struct wg_net, + fib_dispatch_work.work); + struct wg_device *wg; + + rtnl_lock(); + list_for_each_entry(wg, &device_list, device_list) { + if (rcu_access_pointer(wg->creating_net) != wn->net || + wg->transport != WG_TRANSPORT_TCP || + !READ_ONCE(wg->tcp_cleanup_scheduled) || + !netif_running(wg->dev)) + continue; + mod_delayed_work(system_wq, &wg->tcp_route_work, + msecs_to_jiffies(100)); + } + rtnl_unlock(); +} + +static int wg_tcp_fib_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct wg_net *wn = container_of(nb, struct wg_net, fib_notifier); + const struct fib_notifier_info *info = data; + + if (!READ_ONCE(wn->fib_registered) || !info || + (info->family != AF_INET && info->family != AF_INET6)) + return NOTIFY_DONE; + switch (action) { + case FIB_EVENT_ENTRY_REPLACE: + case FIB_EVENT_ENTRY_APPEND: + case FIB_EVENT_ENTRY_ADD: + case FIB_EVENT_ENTRY_DEL: + case FIB_EVENT_RULE_ADD: + case FIB_EVENT_RULE_DEL: + case FIB_EVENT_NH_ADD: + case FIB_EVENT_NH_DEL: + mod_delayed_work(system_wq, &wn->fib_dispatch_work, 0); + break; + default: + break; + } + return NOTIFY_DONE; +} + +/* Address and link notifiers run under RTNL, which also protects device_list. + * Queueing keeps socket shutdown and reconnect work out of notifier context and + * coalesces the event bursts emitted by one administrative change. + */ +static void wg_tcp_schedule_route_change(struct net_device *changed_dev) +{ + struct wg_device *wg; + + if (!changed_dev) + return; + list_for_each_entry(wg, &device_list, device_list) { + if (wg->dev == changed_dev || + rcu_access_pointer(wg->creating_net) != dev_net(changed_dev) || + wg->transport != WG_TRANSPORT_TCP) + continue; + mod_delayed_work(system_wq, &wg->tcp_route_work, + msecs_to_jiffies(100)); + } +} + +static int wg_netdevice_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct net_device *changed_dev = netdev_notifier_info_to_dev(data); + + switch (action) { + case NETDEV_UP: + case NETDEV_DOWN: + case NETDEV_CHANGE: + case NETDEV_CHANGEADDR: + case NETDEV_UNREGISTER: + wg_tcp_schedule_route_change(changed_dev); + break; + default: + break; + } + return NOTIFY_DONE; +} + +static struct notifier_block netdevice_notifier = { + .notifier_call = wg_netdevice_notification +}; + +static int wg_inetaddr_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + const struct in_ifaddr *ifa = data; + + if (ifa && ifa->ifa_dev) + wg_tcp_schedule_route_change(ifa->ifa_dev->dev); + return NOTIFY_DONE; +} + +static struct notifier_block inetaddr_notifier = { + .notifier_call = wg_inetaddr_notification +}; + +#if IS_ENABLED(CONFIG_IPV6) +static int wg_inet6addr_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + const struct inet6_ifaddr *ifa = data; + + if (ifa && ifa->idev) + wg_tcp_schedule_route_change(ifa->idev->dev); + return NOTIFY_DONE; +} + +static struct notifier_block inet6addr_notifier = { + .notifier_call = wg_inet6addr_notification +}; +#endif + static int wg_stop(struct net_device *dev) { struct wg_device *wg = netdev_priv(dev); struct wg_peer *peer; struct sk_buff *skb; + wg_dbg("Entering wg_stop: dev=%px\n", dev); + WRITE_ONCE(wg->tcp_cleanup_scheduled, false); + cancel_delayed_work_sync(&wg->tcp_route_work); mutex_lock(&wg->device_update_lock); + if (wg->transport == WG_TRANSPORT_TCP) { + /* Quiesce every connect/removal owner before releasing the shared + * listeners. Otherwise an in-flight connect can republish listener or + * peer socket state after the device teardown pass. + */ + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_tcp_peer_stop(peer); + wg_tcp_listener_socket_release(wg); + } + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + wg_destruct_tcp_connection_list(wg); + /* Destruction drains temp-peer callbacks that may have passed their + * cleanup flag check before shutdown. Catch any device work queued by + * such a callback after the first cancellation pass. + */ + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + list_for_each_entry(peer, &wg->peer_list, peer_list) { wg_packet_purge_staged_packets(peer); wg_timers_stop(peer); @@ -129,8 +366,12 @@ static int wg_stop(struct net_device *dev) mutex_unlock(&wg->device_update_lock); while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL) kfree_skb(skb); + atomic_set(&wg->handshake_queue_len, 0); + wg_socket_reinit(wg, NULL, NULL); + + wg_dbg("Exiting wg_stop: dev=%px\n", dev); return 0; } @@ -144,6 +385,8 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) u32 mtu; int ret; + wg_dbg("Entering wg_xmit: skb=%px, dev=%px\n", skb, dev); + if (unlikely(!wg_check_packet_protocol(skb))) { ret = -EPROTONOSUPPORT; net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); @@ -218,6 +461,7 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) wg_packet_send_staged_packets(peer); wg_peer_put(peer); + wg_dbg("Exiting wg_xmit: skb=%px, dev=%px\n", skb, dev); return NETDEV_TX_OK; err_peer: @@ -230,6 +474,7 @@ err_icmp: err: DEV_STATS_INC(dev, tx_errors); kfree_skb(skb); + wg_dbg("Exiting wg_xmit with error: skb=%px, dev=%px, ret=%d\n", skb, dev, ret); return ret; } @@ -244,10 +489,28 @@ static void wg_destruct(struct net_device *dev) { struct wg_device *wg = netdev_priv(dev); + wg_dbg("Entering wg_destruct: dev=%px\n", dev); + rtnl_lock(); list_del(&wg->device_list); rtnl_unlock(); + cancel_delayed_work_sync(&wg->tcp_route_work); mutex_lock(&wg->device_update_lock); + WRITE_ONCE(wg->tcp_cleanup_scheduled, false); + if (wg->transport == WG_TRANSPORT_TCP) { + struct wg_peer *peer; + + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_tcp_peer_stop(peer); + wg_tcp_listener_socket_release(wg); + } + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + wg_destruct_tcp_connection_list(wg); + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + if (wg->tcp_auth_wq) { + destroy_workqueue(wg->tcp_auth_wq); + wg->tcp_auth_wq = NULL; + } rcu_assign_pointer(wg->creating_net, NULL); wg->incoming_port = 0; wg_socket_reinit(wg, NULL, NULL); @@ -269,6 +532,8 @@ static void wg_destruct(struct net_device *dev) pr_debug("%s: Interface destroyed\n", dev->name); free_netdev(dev); + + wg_dbg("Exiting wg_destruct: dev=%px\n", dev); } static const struct device_type device_type = { .name = KBUILD_MODNAME }; @@ -280,13 +545,16 @@ static void wg_setup(struct net_device *dev) NETIF_F_SG | NETIF_F_GSO | NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) + - max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); + max(sizeof(struct ipv6hdr), sizeof(struct iphdr)) + + (wg->transport == WG_TRANSPORT_TCP ? WG_TCP_ENCAP_HDR_LEN : 0); + + wg_dbg("Entering wg_setup: dev=%px\n", dev); dev->netdev_ops = &netdev_ops; dev->header_ops = &ip_tunnel_header_ops; dev->hard_header_len = 0; dev->addr_len = 0; - dev->needed_headroom = DATA_PACKET_HEAD_ROOM; + dev->needed_headroom = DATA_PACKET_HEAD_ROOM + (wg->transport ? WG_TCP_ENCAP_HDR_LEN : 0); dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); dev->type = ARPHRD_NONE; dev->flags = IFF_POINTOPOINT | IFF_NOARP; @@ -305,6 +573,8 @@ static void wg_setup(struct net_device *dev) memset(wg, 0, sizeof(*wg)); wg->dev = dev; + + wg_dbg("Exiting wg_setup: dev=%px\n", dev); } static int wg_newlink(struct net *src_net, struct net_device *dev, @@ -314,6 +584,8 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, struct wg_device *wg = netdev_priv(dev); int ret = -ENOMEM; + wg_dbg("Entering wg_newlink: src_net=%px, dev=%px, tb=%px, data=%px, extack=%px\n", src_net, dev, tb, data, extack); + rcu_assign_pointer(wg->creating_net, src_net); init_rwsem(&wg->static_identity.lock); mutex_init(&wg->socket_update_lock); @@ -322,6 +594,13 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, wg_cookie_checker_init(&wg->cookie_checker, wg); INIT_LIST_HEAD(&wg->peer_list); wg->device_update_gen = 1; + /* Initialize the tcp_cleanup_scheduled flag and spinlock */ + wg->tcp_cleanup_scheduled = false; + spin_lock_init(&wg->tcp_cleanup_lock); + + /* Initialize the work for tcp_cleanup_worker */ + INIT_DELAYED_WORK(&wg->tcp_cleanup_work, wg_tcp_cleanup_worker); + INIT_DELAYED_WORK(&wg->tcp_route_work, wg_tcp_route_change_worker); wg->peer_hashtable = wg_pubkey_hashtable_alloc(); if (!wg->peer_hashtable) @@ -375,14 +654,22 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, list_add(&wg->device_list, &device_list); + INIT_LIST_HEAD(&wg->tcp_connection_list); + spin_lock_init(&wg->tcp_connection_list_lock); + spin_lock_init(&wg->tcp_accept_lock); + atomic64_set(&wg->tcp_connection_sequence, 0); + wg->tcp_socket4_ready = false; + wg->tcp_socket6_ready = false; + /* We wait until the end to assign priv_destructor, so that * register_netdevice doesn't call it for us if it fails. */ dev->priv_destructor = wg_destruct; pr_debug("%s: Interface created\n", dev->name); - return ret; + wg_dbg("Exiting wg_newlink: src_net=%px, dev=%px, ret=%d\n", src_net, dev, ret); + return ret; err_uninit_ratelimiter: wg_ratelimiter_uninit(); err_free_handshake_queue: @@ -403,6 +690,7 @@ err_free_index_hashtable: kvfree(wg->index_hashtable); err_free_peer_hashtable: kvfree(wg->peer_hashtable); + wg_dbg("Exiting wg_newlink with error: src_net=%px, dev=%px, ret=%d\n", src_net, dev, ret); return ret; } @@ -413,17 +701,57 @@ static struct rtnl_link_ops link_ops __read_mostly = { .newlink = wg_newlink, }; +static int wg_netns_init(struct net *net) +{ + struct wg_net *wn = net_generic(net, wg_net_id); + int ret; + + wn->net = net; + wn->fib_notifier.notifier_call = wg_tcp_fib_notification; + INIT_DELAYED_WORK(&wn->fib_dispatch_work, + wg_tcp_fib_dispatch_worker); + ret = register_fib_notifier(net, &wn->fib_notifier, NULL, NULL); + if (ret) { + pr_warn("wireguard: TCP route notifications unavailable in netns %u: %d\n", + net->ns.inum, ret); + return 0; + } + WRITE_ONCE(wn->fib_registered, true); + return 0; +} + static void wg_netns_pre_exit(struct net *net) { + struct wg_net *wn = net_generic(net, wg_net_id); struct wg_device *wg; struct wg_peer *peer; + wg_dbg("Entering wg_netns_pre_exit: net=%px\n", net); + if (READ_ONCE(wn->fib_registered)) { + WRITE_ONCE(wn->fib_registered, false); + unregister_fib_notifier(net, &wn->fib_notifier); + } + cancel_delayed_work_sync(&wn->fib_dispatch_work); + rtnl_lock(); list_for_each_entry(wg, &device_list, device_list) { if (rcu_access_pointer(wg->creating_net) == net) { pr_debug("%s: Creating namespace exiting\n", wg->dev->name); netif_carrier_off(wg->dev); + cancel_delayed_work_sync(&wg->tcp_route_work); mutex_lock(&wg->device_update_lock); + if (wg->transport == WG_TRANSPORT_TCP) { + /* Stop every user of sockets created in this namespace + * before publishing that the namespace is gone. + */ + WRITE_ONCE(wg->tcp_cleanup_scheduled, false); + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_tcp_peer_stop(peer); + wg_tcp_listener_socket_release(wg); + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + wg_destruct_tcp_connection_list(wg); + cancel_delayed_work_sync(&wg->tcp_cleanup_work); + } rcu_assign_pointer(wg->creating_net, NULL); wg_socket_reinit(wg, NULL, NULL); list_for_each_entry(peer, &wg->peer_list, peer_list) @@ -432,19 +760,26 @@ static void wg_netns_pre_exit(struct net *net) } } rtnl_unlock(); + + wg_dbg("Exiting wg_netns_pre_exit: net=%px\n", net); } static struct pernet_operations pernet_ops = { - .pre_exit = wg_netns_pre_exit + .init = wg_netns_init, + .pre_exit = wg_netns_pre_exit, + .id = &wg_net_id, + .size = sizeof(struct wg_net), }; int __init wg_device_init(void) { int ret; + wg_dbg("Entering wg_device_init\n"); + ret = register_pm_notifier(&pm_notifier); if (ret) - return ret; + goto error; ret = register_random_vmfork_notifier(&vm_notifier); if (ret) @@ -454,26 +789,60 @@ int __init wg_device_init(void) if (ret) goto error_vm; - ret = rtnl_link_register(&link_ops); + ret = register_netdevice_notifier(&netdevice_notifier); if (ret) goto error_pernet; + ret = register_inetaddr_notifier(&inetaddr_notifier); + if (ret) + goto error_netdevice; + +#if IS_ENABLED(CONFIG_IPV6) + ret = register_inet6addr_notifier(&inet6addr_notifier); + if (ret) + goto error_inetaddr; +#endif + + ret = rtnl_link_register(&link_ops); + if (ret) + goto error_inet6addr; + + wg_dbg("Exiting wg_device_init: ret=0\n"); return 0; +error_inet6addr: +#if IS_ENABLED(CONFIG_IPV6) + unregister_inet6addr_notifier(&inet6addr_notifier); +error_inetaddr: +#endif + unregister_inetaddr_notifier(&inetaddr_notifier); +error_netdevice: + unregister_netdevice_notifier(&netdevice_notifier); error_pernet: unregister_pernet_device(&pernet_ops); error_vm: unregister_random_vmfork_notifier(&vm_notifier); error_pm: unregister_pm_notifier(&pm_notifier); +error: + wg_dbg("Exiting wg_device_init with error: ret=%d\n", ret); return ret; } void wg_device_uninit(void) { + wg_dbg("Entering wg_device_uninit\n"); + rtnl_link_unregister(&link_ops); +#if IS_ENABLED(CONFIG_IPV6) + unregister_inet6addr_notifier(&inet6addr_notifier); +#endif + unregister_inetaddr_notifier(&inetaddr_notifier); + unregister_netdevice_notifier(&netdevice_notifier); unregister_pernet_device(&pernet_ops); unregister_random_vmfork_notifier(&vm_notifier); unregister_pm_notifier(&pm_notifier); rcu_barrier(); + + wg_dbg("Exiting wg_device_uninit\n"); } diff --git a/kernel/device.h b/kernel/device.h index 43c7cebbf50b08f2a1868f0017d0bee8aee700f8..8abb4df7ae5301565f4c4d499d87684a3600bc2d 100644 --- a/kernel/device.h +++ b/kernel/device.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ #ifndef _WG_DEVICE_H @@ -11,12 +12,15 @@ #include "peerlookup.h" #include "cookie.h" + #include #include #include +#include #include #include #include +#include struct wg_device; @@ -33,27 +37,76 @@ struct crypt_queue { struct prev_queue { struct sk_buff *head, *tail, *peeked; - struct { struct sk_buff *next, *prev; } empty; // Match first 2 members of struct sk_buff. + /* Must match the first two members of struct sk_buff. */ + struct { struct sk_buff *next, *prev; } empty; atomic_t count; }; +struct endpoint { + union { + struct sockaddr addr; + struct sockaddr_in addr4; + struct sockaddr_in6 addr6; + }; + union { + struct { + struct in_addr src4; + /* Essentially the same as addr6->scope_id */ + int src_if4; + }; + struct in6_addr src6; + }; +}; + +#define WG_TCP_ACCEPT_SOURCE_SLOTS 128 + +struct wg_tcp_accept_source { + union { + __be32 addr4; + struct in6_addr addr6; + } address; + unsigned long window_started; + unsigned long last_seen; + u32 scope_id; + u16 accepts; + sa_family_t family; +}; + struct wg_device { struct net_device *dev; struct crypt_queue encrypt_queue, decrypt_queue, handshake_queue; - struct sock __rcu *sock4, *sock6; + struct sock __rcu *sock4, *sock6; /* UDP listening sockets */ + struct socket __rcu *tcp_listen_socket4, *tcp_listen_socket6; /* TCP listening sockets */ struct net __rcu *creating_net; struct noise_static_identity static_identity; struct workqueue_struct *packet_crypt_wq,*handshake_receive_wq, *handshake_send_wq; + struct workqueue_struct *tcp_auth_wq; struct cookie_checker cookie_checker; struct pubkey_hashtable *peer_hashtable; struct index_hashtable *index_hashtable; struct allowedips peer_allowedips; struct mutex device_update_lock, socket_update_lock; - struct list_head device_list, peer_list; + struct endpoint device_endpoint; + struct list_head device_list, peer_list, tcp_connection_list; + struct task_struct *tcp_listener4_thread, *tcp_listener6_thread; + struct delayed_work tcp_cleanup_work; + struct delayed_work tcp_route_work; + spinlock_t tcp_cleanup_lock; /* Add a spinlock to protect the flag */ + bool tcp_cleanup_scheduled; + bool tcp_socket4_ready; + bool tcp_socket6_ready; + bool listener_active; + spinlock_t tcp_connection_list_lock; + unsigned int tcp_pending_connections; + unsigned int tcp_tracked_connections; + atomic64_t tcp_connection_sequence; + spinlock_t tcp_accept_lock; + struct wg_tcp_accept_source tcp_accept_sources[WG_TCP_ACCEPT_SOURCE_SLOTS]; atomic_t handshake_queue_len; unsigned int num_peers, device_update_gen; u32 fwmark; u16 incoming_port; + u8 transport; }; int wg_device_init(void); diff --git a/kernel/main.c b/kernel/main.c index ee4da9ab8013c3ad2721e0e1d4432b2fe007886b..38b4db7868505c439fc48af9808f8207991d0663 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -9,6 +9,8 @@ #include "queueing.h" #include "ratelimiter.h" #include "netlink.h" +#include "socket.h" +#include "cookie.h" #include @@ -17,37 +19,50 @@ #include #include +#include "wg_tcp_debug.h" + static int __init wg_mod_init(void) { int ret; + wg_dbg("Entering: wg_mod_init\n"); + ret = wg_allowedips_slab_init(); + wg_dbg("wg_mod_init: wg_allowedips_slab_init() = %d\n", ret); if (ret < 0) goto err_allowedips; #ifdef DEBUG ret = -ENOTRECOVERABLE; if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() || - !wg_ratelimiter_selftest()) + !wg_ratelimiter_selftest() || !wg_cookie_policy_selftest()) { + wg_dbg("wg_mod_init: Self-test failed\n"); goto err_peer; + } #endif wg_noise_init(); + wg_dbg("wg_mod_init: wg_noise_init() completed\n"); ret = wg_peer_init(); + wg_dbg("wg_mod_init: wg_peer_init() = %d\n", ret); if (ret < 0) goto err_peer; ret = wg_device_init(); + wg_dbg("wg_mod_init: wg_device_init() = %d\n", ret); if (ret < 0) goto err_device; ret = wg_genetlink_init(); + wg_dbg("wg_mod_init: wg_genetlink_init() = %d\n", ret); if (ret < 0) goto err_netlink; - pr_info("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n"); - pr_info("Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved.\n"); + wg_dbg("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n"); + wg_dbg("Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved.\n"); + wg_dbg("TCP Transport Mode - Copyright (C) 2024 Jeff Nathan and Dragos Ruiu. All Rights Reserved.\n"); + wg_dbg("Exiting: wg_mod_init\n"); return 0; err_netlink: @@ -57,21 +72,33 @@ err_device: err_peer: wg_allowedips_slab_uninit(); err_allowedips: + wg_dbg("Exiting with error: wg_mod_init, ret = %d\n", ret); return ret; } static void __exit wg_mod_exit(void) { + wg_dbg("Entering: wg_mod_exit\n"); + wg_genetlink_uninit(); + wg_dbg("wg_mod_exit: wg_genetlink_uninit() completed\n"); + wg_device_uninit(); + wg_dbg("wg_mod_exit: wg_device_uninit() completed\n"); + wg_peer_uninit(); + wg_dbg("wg_mod_exit: wg_peer_uninit() completed\n"); + wg_allowedips_slab_uninit(); + wg_dbg("wg_mod_exit: wg_allowedips_slab_uninit() completed\n"); + + wg_dbg("Exiting: wg_mod_exit\n"); } module_init(wg_mod_init); module_exit(wg_mod_exit); MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("WireGuard secure network tunnel"); +MODULE_DESCRIPTION("WireGuard secure network tunnel - with UDP/TCP"); MODULE_AUTHOR("Jason A. Donenfeld "); MODULE_VERSION(WIREGUARD_VERSION); MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME); diff --git a/kernel/netlink.c b/kernel/netlink.c index e220d761b1f27aa31eab3ad1b9211ddfe55eaabd..a37da59335f8321e0f6529b4580a9ebdb9780d8e 100644 --- a/kernel/netlink.c +++ b/kernel/netlink.c @@ -1,21 +1,27 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ #include "netlink.h" #include "device.h" #include "peer.h" #include "socket.h" +#include "wg_tcp.h" #include "queueing.h" #include "messages.h" #include #include +#include #include +#include #include +#include #include +#include "wg_tcp_debug.h" static struct genl_family genl_family; @@ -27,7 +33,8 @@ static const struct nla_policy device_policy[WGDEVICE_A_MAX + 1] = { [WGDEVICE_A_FLAGS] = { .type = NLA_U32 }, [WGDEVICE_A_LISTEN_PORT] = { .type = NLA_U16 }, [WGDEVICE_A_FWMARK] = { .type = NLA_U32 }, - [WGDEVICE_A_PEERS] = { .type = NLA_NESTED } + [WGDEVICE_A_PEERS] = { .type = NLA_NESTED }, + [WGDEVICE_A_TRANSPORT] = { .type = NLA_U8 } }; static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = { @@ -46,22 +53,304 @@ static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = { static const struct nla_policy allowedip_policy[WGALLOWEDIP_A_MAX + 1] = { [WGALLOWEDIP_A_FAMILY] = { .type = NLA_U16 }, [WGALLOWEDIP_A_IPADDR] = NLA_POLICY_MIN_LEN(sizeof(struct in_addr)), - [WGALLOWEDIP_A_CIDR_MASK] = { .type = NLA_U8 } + [WGALLOWEDIP_A_CIDR_MASK] = { .type = NLA_U8 }, + [WGALLOWEDIP_A_FLAGS] = { .type = NLA_U32 } }; -static struct wg_device *lookup_interface(struct nlattr **attrs, - struct sk_buff *skb) + +#ifdef DIAGNOSTIC +/* Diagnostic functions for decoding netlink attributes and messages */ + +/* Function to print the libmnl formatted netlink message header */ +static void wg_print_netlink_header_libmnl(const struct nlmsghdr *nlh) +{ + wg_dbg("----------------\t------------------\n"); + wg_dbg("| %.010u |\t| message length |\n", nlh->nlmsg_len); + wg_dbg("| %.05u | %c%c%c%c |\t| type | flags |\n", + nlh->nlmsg_type, + nlh->nlmsg_flags & NLM_F_REQUEST ? 'R' : '-', + nlh->nlmsg_flags & NLM_F_MULTI ? 'M' : '-', + nlh->nlmsg_flags & NLM_F_ACK ? 'A' : '-', + nlh->nlmsg_flags & NLM_F_ECHO ? 'E' : '-'); + wg_dbg("| %.010u |\t| sequence number|\n", nlh->nlmsg_seq); + wg_dbg("| %.010u |\t| port ID |\n", nlh->nlmsg_pid); + wg_dbg("----------------\t------------------\n"); +} + +/* Function to print the libmnl formatted netlink message payload */ +static void wg_print_netlink_payload_libmnl(const struct nlmsghdr *nlh, size_t extra_header_size) +{ + unsigned int i; + int rem = 0; + + for (i = sizeof(struct nlmsghdr); i < nlh->nlmsg_len; i += 4) { + char *b = (char *)nlh; + struct nlattr *attr = (struct nlattr *)(b + i); + + if (nlh->nlmsg_type < NLMSG_MIN_TYPE) { + wg_dbg("| %.2x %.2x %.2x %.2x |\t", + 0xff & b[i], 0xff & b[i + 1], + 0xff & b[i + 2], 0xff & b[i + 3]); + wg_dbg("| |\n"); + } else if (extra_header_size > 0) { + extra_header_size -= 4; + wg_dbg("| %.2x %.2x %.2x %.2x |\t", + 0xff & b[i], 0xff & b[i + 1], + 0xff & b[i + 2], 0xff & b[i + 3]); + wg_dbg("| extra header |\n"); + } else if (rem == 0 && (attr->nla_type & NLA_TYPE_MASK) != 0) { + wg_dbg("|%.5u|%c%c|%.5u|\t", + attr->nla_len, + attr->nla_type & NLA_F_NESTED ? 'N' : '-', + attr->nla_type & NLA_F_NET_BYTEORDER ? 'B' : '-', + attr->nla_type & NLA_TYPE_MASK); + wg_dbg("|len |flags| type|\n"); + + if (!(attr->nla_type & NLA_F_NESTED)) { + rem = NLA_ALIGN(attr->nla_len) - sizeof(struct nlattr); + } + } else if (rem > 0) { + rem -= 4; + wg_dbg("| %.2x %.2x %.2x %.2x |\t", + 0xff & b[i], 0xff & b[i + 1], + 0xff & b[i + 2], 0xff & b[i + 3]); + wg_dbg("| data |"); + wg_dbg("\t %c %c %c %c\n", + isprint(b[i]) ? b[i] : ' ', + isprint(b[i + 1]) ? b[i + 1] : ' ', + isprint(b[i + 2]) ? b[i + 2] : ' ', + isprint(b[i + 3]) ? b[i + 3] : ' '); + } + } + wg_dbg("----------------\t------------------\n"); +} + +/* Print the netlink message using libmnl format */ +static void wg_print_netlink_message_libmnl(const struct nlmsghdr *nlh) +{ + wg_print_netlink_header_libmnl(nlh); + wg_print_netlink_payload_libmnl(nlh, 0); +} + +/* Routine to parse and print flags with verbose values */ +static void wg_print_flags_verbose(uint32_t flags) +{ + wg_dbg("Flags: 0x%08x (", flags); + if (flags & NLM_F_REQUEST) wg_dbg("REQUEST "); + if (flags & NLM_F_MULTI) wg_dbg("MULTI "); + if (flags & NLM_F_ACK) wg_dbg("ACK "); + if (flags & NLM_F_ECHO) wg_dbg("ECHO "); + if (flags & NLM_F_REPLACE) wg_dbg("REPLACE "); + if (flags & NLM_F_EXCL) wg_dbg("EXCL "); + if (flags & NLM_F_CREATE) wg_dbg("CREATE "); + if (flags & NLM_F_APPEND) wg_dbg("APPEND "); + wg_dbg(")\n"); +} + +/* Routine to parse and print peer flags with verbose labels */ +static void wg_print_peer_flags_verbose(uint32_t flags) +{ + wg_dbg("Peer Flags: 0x%08x (", flags); + if (flags & WGPEER_F_REMOVE_ME) wg_dbg("REMOVE_ME "); + if (flags & WGPEER_F_REPLACE_ALLOWEDIPS) wg_dbg("REPLACE_ALLOWEDIPS "); + if (flags & WGPEER_F_UPDATE_ONLY) wg_dbg("UPDATE_ONLY "); + wg_dbg(")\n"); +} + +/* Functions to print the allowed IP attributes */ +static void wg_print_allowedip_attr(const struct nlattr *attr) +{ + int type = nla_type(attr); + + switch (type) { + case WGALLOWEDIP_A_FAMILY: + wg_dbg("WGALLOWEDIP_A_FAMILY: %u\n", nla_get_u16(attr)); + break; + case WGALLOWEDIP_A_IPADDR: + wg_dbg("WGALLOWEDIP_A_IPADDR: %pI6\n", nla_data(attr)); + break; + case WGALLOWEDIP_A_CIDR_MASK: + wg_dbg("WGALLOWEDIP_A_CIDR_MASK: %u\n", nla_get_u8(attr)); + break; + default: + wg_dbg("Unknown Allowed IP Attribute Type: %d\n", type); + break; + } +} + +static void wg_print_peer_allowedips(const struct nlattr *attr) +{ + struct nlattr *nested_attr; + int rem; + + nla_for_each_nested(nested_attr, attr, rem) { + wg_print_allowedip_attr(nested_attr); + } +} + +/* Functions to print the peer attributes */ +static void wg_print_peer_attr(const struct nlattr *attr) +{ + int type = nla_type(attr); + + switch (type) { + case WGPEER_A_PUBLIC_KEY: + wg_dbg("WGPEER_A_PUBLIC_KEY: %*phN\n", nla_len(attr), nla_data(attr)); + break; + case WGPEER_A_PRESHARED_KEY: + wg_dbg("WGPEER_A_PRESHARED_KEY: %*phN\n", nla_len(attr), nla_data(attr)); + break; + case WGPEER_A_FLAGS: + wg_dbg("WGPEER_A_FLAGS: %u\n", nla_get_u32(attr)); + wg_print_peer_flags_verbose(nla_get_u32(attr)); + break; + case WGPEER_A_ENDPOINT: + wg_dbg("WGPEER_A_ENDPOINT: %pIS\n", nla_data(attr)); + break; + case WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: + wg_dbg("WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: %u\n", nla_get_u16(attr)); + break; + case WGPEER_A_LAST_HANDSHAKE_TIME: { + const struct __kernel_timespec *ts = nla_data(attr); + wg_dbg("WGPEER_A_LAST_HANDSHAKE_TIME: %lld.%.9ld\n", + (long long)ts->tv_sec, ts->tv_nsec); + break; + } + case WGPEER_A_RX_BYTES: + wg_dbg("WGPEER_A_RX_BYTES: %llu\n", (unsigned long long)nla_get_u64(attr)); + break; + case WGPEER_A_TX_BYTES: + wg_dbg("WGPEER_A_TX_BYTES: %llu\n", (unsigned long long)nla_get_u64(attr)); + break; + case WGPEER_A_ALLOWEDIPS: + wg_dbg("WGPEER_A_ALLOWEDIPS (Nested Attributes):\n"); + wg_print_peer_allowedips(attr); + break; + default: + wg_dbg("Unknown Peer Attribute Type: %d\n", type); + break; + } +} + +/* Functions to print the device attributes */ +static void wg_print_device_peers(const struct nlattr *attr) +{ + struct nlattr *nested_attr; + int rem; + + nla_for_each_nested(nested_attr, attr, rem) { + wg_print_peer_attr(nested_attr); + } +} + +static void wg_print_device_attr(const struct nlattr *attr) +{ + int type = nla_type(attr); + + switch (type) { + case WGDEVICE_A_IFINDEX: + wg_dbg("WGDEVICE_A_IFINDEX: %u\n", nla_get_u32(attr)); + break; + case WGDEVICE_A_IFNAME: + wg_dbg("WGDEVICE_A_IFNAME: %s\n", nla_data(attr)); + break; + case WGDEVICE_A_PRIVATE_KEY: + wg_dbg("WGDEVICE_A_PRIVATE_KEY: %*phN\n", nla_len(attr), nla_data(attr)); + break; + case WGDEVICE_A_PUBLIC_KEY: + wg_dbg("WGDEVICE_A_PUBLIC_KEY: %*phN\n", nla_len(attr), nla_data(attr)); + break; + case WGDEVICE_A_FLAGS: + wg_dbg("WGDEVICE_A_FLAGS: %u\n", nla_get_u32(attr)); + break; + case WGDEVICE_A_LISTEN_PORT: + wg_dbg("WGDEVICE_A_LISTEN_PORT: %u\n", nla_get_u16(attr)); + break; + case WGDEVICE_A_FWMARK: + wg_dbg("WGDEVICE_A_FWMARK: %u\n", nla_get_u32(attr)); + break; + case WGDEVICE_A_PEERS: + wg_dbg("WGDEVICE_A_PEERS (Nested Attributes):\n"); + wg_print_device_peers(attr); + break; + case WGDEVICE_A_TRANSPORT: + wg_dbg("WGDEVICE_A_TRANSPORT: %u\n", nla_get_u8(attr)); + break; + default: + wg_dbg("Unknown Device Attribute Type: %d\n", type); + break; + } +} + +static void wg_print_netlink_message_verbose(const struct nlmsghdr *nlh) +{ + struct nlattr *attr; + int rem; + + wg_dbg("Verbose Netlink Message:\n"); + wg_dbg(" nlmsg_len: %u\n", nlh->nlmsg_len); + wg_dbg(" nlmsg_type: %u\n", nlh->nlmsg_type); + + /* Print command */ + switch (nlh->nlmsg_type) { + case WG_CMD_GET_DEVICE: + wg_dbg(" Command: WG_CMD_GET_DEVICE\n"); + break; + case WG_CMD_SET_DEVICE: + wg_dbg(" Command: WG_CMD_SET_DEVICE\n"); + break; + default: + wg_dbg(" Unknown Command: %u\n", nlh->nlmsg_type); + break; + } + + wg_print_flags_verbose(nlh->nlmsg_flags); + wg_dbg(" nlmsg_seq: %u\n", nlh->nlmsg_seq); + wg_dbg(" nlmsg_pid: %u\n", nlh->nlmsg_pid); + wg_dbg("Attributes:\n"); + + nla_for_each_attr(attr, nlmsg_data(nlh), nlmsg_len(nlh) - NLMSG_HDRLEN, rem) { + switch (nlh->nlmsg_type) { + case WG_CMD_GET_DEVICE: + case WG_CMD_SET_DEVICE: + wg_print_device_attr(attr); + break; + default: + wg_dbg("Unknown Netlink Message Type: %u\n", nlh->nlmsg_type); + break; + } + } + + wg_dbg("Hex Dump of Netlink Message:\n"); + wg_dbg("%*phN\n", nlh->nlmsg_len, nlh); +} + +/* Add a function to print the entire buffer of netlink messages */ +static void wg_print_netlink_buffer(const void *buf, size_t len) +{ + const struct nlmsghdr *nlh = buf; + + while (nlh && NLMSG_OK(nlh, len)) { + wg_print_netlink_message_verbose(nlh); + wg_print_netlink_message_libmnl(nlh); + nlh = NLMSG_NEXT(nlh, len); + } +} + +#endif /* DIAGNOSTIC */ + +static struct wg_device *lookup_interface(struct nlattr **attrs, struct sk_buff *skb) { struct net_device *dev = NULL; + wg_dbg("Entering lookup_interface: attrs = %px, skb = %px\n", attrs, skb); + if (!attrs[WGDEVICE_A_IFINDEX] == !attrs[WGDEVICE_A_IFNAME]) return ERR_PTR(-EBADR); if (attrs[WGDEVICE_A_IFINDEX]) - dev = dev_get_by_index(sock_net(skb->sk), - nla_get_u32(attrs[WGDEVICE_A_IFINDEX])); + dev = dev_get_by_index(sock_net(skb->sk), nla_get_u32(attrs[WGDEVICE_A_IFINDEX])); else if (attrs[WGDEVICE_A_IFNAME]) - dev = dev_get_by_name(sock_net(skb->sk), - nla_data(attrs[WGDEVICE_A_IFNAME])); + dev = dev_get_by_name(sock_net(skb->sk), nla_data(attrs[WGDEVICE_A_IFNAME])); if (!dev) return ERR_PTR(-ENODEV); if (!dev->rtnl_link_ops || !dev->rtnl_link_ops->kind || @@ -69,27 +358,33 @@ static struct wg_device *lookup_interface(struct nlattr **attrs, dev_put(dev); return ERR_PTR(-EOPNOTSUPP); } + + wg_dbg("Exiting lookup_interface\n"); + return netdev_priv(dev); } -static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr, - int family) +static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr, int family) { struct nlattr *allowedip_nest; + wg_dbg("Entering get_allowedips: skb = %px, ip = %px, cidr = %u, family = %d\n", skb, ip, cidr, family); + allowedip_nest = nla_nest_start(skb, 0); if (!allowedip_nest) return -EMSGSIZE; if (nla_put_u8(skb, WGALLOWEDIP_A_CIDR_MASK, cidr) || nla_put_u16(skb, WGALLOWEDIP_A_FAMILY, family) || - nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ? - sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) { + nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ? sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) { nla_nest_cancel(skb, allowedip_nest); return -EMSGSIZE; } nla_nest_end(skb, allowedip_nest); + + wg_dbg("Exiting get_allowedips\n"); + return 0; } @@ -102,20 +397,19 @@ struct dump_ctx { #define DUMP_CTX(cb) ((struct dump_ctx *)(cb)->args) -static int -get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) +static int get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) { - struct nlattr *allowedips_nest, *peer_nest = nla_nest_start(skb, 0); struct allowedips_node *allowedips_node = ctx->next_allowedip; bool fail; + wg_dbg("Entering get_peer: peer = %px, skb = %px, ctx = %px\n", peer, skb, ctx); + if (!peer_nest) return -EMSGSIZE; down_read(&peer->handshake.lock); - fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, - peer->handshake.remote_static); + fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, peer->handshake.remote_static); up_read(&peer->handshake.lock); if (fail) goto err; @@ -127,39 +421,42 @@ get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) }; down_read(&peer->handshake.lock); - fail = nla_put(skb, WGPEER_A_PRESHARED_KEY, - NOISE_SYMMETRIC_KEY_LEN, - peer->handshake.preshared_key); + fail = nla_put(skb, WGPEER_A_PRESHARED_KEY, NOISE_SYMMETRIC_KEY_LEN, peer->handshake.preshared_key); up_read(&peer->handshake.lock); if (fail) goto err; - if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME, - sizeof(last_handshake), &last_handshake) || - nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, - peer->persistent_keepalive_interval) || - nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes, - WGPEER_A_UNSPEC) || - nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes, - WGPEER_A_UNSPEC) || + if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME, sizeof(last_handshake), &last_handshake) || + nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, peer->persistent_keepalive_interval) || + nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes, WGPEER_A_UNSPEC) || + nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes, WGPEER_A_UNSPEC) || nla_put_u32(skb, WGPEER_A_PROTOCOL_VERSION, 1)) goto err; read_lock_bh(&peer->endpoint_lock); - if (peer->endpoint.addr.sa_family == AF_INET) + if (peer->device->transport == WG_TRANSPORT_TCP && + peer->peer_endpoint_set) { + if (peer->peer_endpoint.addr.sa_family == AF_INET) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->peer_endpoint.addr4), + &peer->peer_endpoint.addr4); + else if (peer->peer_endpoint.addr.sa_family == AF_INET6) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->peer_endpoint.addr6), + &peer->peer_endpoint.addr6); + } else if (peer->endpoint.addr.sa_family == AF_INET) { fail = nla_put(skb, WGPEER_A_ENDPOINT, sizeof(peer->endpoint.addr4), &peer->endpoint.addr4); - else if (peer->endpoint.addr.sa_family == AF_INET6) + } else if (peer->endpoint.addr.sa_family == AF_INET6) { fail = nla_put(skb, WGPEER_A_ENDPOINT, sizeof(peer->endpoint.addr6), &peer->endpoint.addr6); + } read_unlock_bh(&peer->endpoint_lock); if (fail) goto err; - allowedips_node = - list_first_entry_or_null(&peer->allowedips_list, - struct allowedips_node, peer_list); + allowedips_node = list_first_entry_or_null(&peer->allowedips_list, struct allowedips_node, peer_list); } if (!allowedips_node) goto no_allowedips; @@ -172,9 +469,8 @@ get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) if (!allowedips_nest) goto err; - list_for_each_entry_from(allowedips_node, &peer->allowedips_list, - peer_list) { - u8 cidr, ip[16] __aligned(__alignof(u64)); + list_for_each_entry_from(allowedips_node, &peer->allowedips_list, peer_list) { + u8 cidr, ip[16] __aligned(__alignof__(u64)); int family; family = wg_allowedips_read_node(allowedips_node, ip, &cidr); @@ -190,6 +486,9 @@ no_allowedips: nla_nest_end(skb, peer_nest); ctx->next_allowedip = NULL; ctx->allowedips_seq = 0; + + wg_dbg("Exiting get_peer\n"); + return 0; err: nla_nest_cancel(skb, peer_nest); @@ -200,10 +499,19 @@ static int wg_get_device_start(struct netlink_callback *cb) { struct wg_device *wg; + wg_dbg("Entering wg_get_device_start: cb = %px\n", cb); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,6,0) wg = lookup_interface(genl_info_dump(cb)->attrs, cb->skb); +#else + wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb); +#endif if (IS_ERR(wg)) return PTR_ERR(wg); DUMP_CTX(cb)->wg = wg; + + wg_dbg("Exiting wg_get_device_start\n"); + return 0; } @@ -217,33 +525,30 @@ static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb) bool done = true; void *hdr; + wg_dbg("Entering wg_get_device_dump: skb = %px, cb = %px\n", skb, cb); + rtnl_lock(); mutex_lock(&wg->device_update_lock); cb->seq = wg->device_update_gen; next_peer_cursor = ctx->next_peer; - hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, - &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE); + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE); if (!hdr) goto out; genl_dump_check_consistent(cb, hdr); if (!ctx->next_peer) { - if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT, - wg->incoming_port) || + if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT, wg->incoming_port) || nla_put_u32(skb, WGDEVICE_A_FWMARK, wg->fwmark) || nla_put_u32(skb, WGDEVICE_A_IFINDEX, wg->dev->ifindex) || - nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name)) + nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name) || + nla_put_u8(skb, WGDEVICE_A_TRANSPORT, wg->transport)) goto out; down_read(&wg->static_identity.lock); if (wg->static_identity.has_identity) { - if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY, - NOISE_PUBLIC_KEY_LEN, - wg->static_identity.static_private) || - nla_put(skb, WGDEVICE_A_PUBLIC_KEY, - NOISE_PUBLIC_KEY_LEN, - wg->static_identity.static_public)) { + if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY, NOISE_PUBLIC_KEY_LEN, wg->static_identity.static_private) || + nla_put(skb, WGDEVICE_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, wg->static_identity.static_public)) { up_read(&wg->static_identity.lock); goto out; } @@ -254,6 +559,7 @@ static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb) peers_nest = nla_nest_start(skb, WGDEVICE_A_PEERS); if (!peers_nest) goto out; + ret = 0; /* If the last cursor was removed via list_del_init in peer_remove, then * we just treat this the same as there being no more peers left. The @@ -290,9 +596,15 @@ out: genlmsg_end(skb, hdr); if (done) { ctx->next_peer = NULL; + + wg_dbg("Exiting wg_get_device_dump\n"); + return 0; } ctx->next_peer = next_peer_cursor; + + wg_dbg("Exiting wg_get_device_dump\n"); + return skb->len; /* At this point, we can't really deal ourselves with safely zeroing out @@ -305,9 +617,14 @@ static int wg_get_device_done(struct netlink_callback *cb) { struct dump_ctx *ctx = DUMP_CTX(cb); + wg_dbg("Entering wg_get_device_done: cb = %px\n", cb); + if (ctx->wg) dev_put(ctx->wg->dev); wg_peer_put(ctx->next_peer); + + wg_dbg("Exiting wg_get_device_done\n"); + return 0; } @@ -315,41 +632,63 @@ static int set_port(struct wg_device *wg, u16 port) { struct wg_peer *peer; + wg_dbg("Entering set_port: wg = %px, port = %u\n", wg, port); + if (wg->incoming_port == port) return 0; + /* Replacing both TCP listeners is not transactional. Require a down/up + * cycle and leave the active listeners completely untouched. + */ + if (wg->transport == WG_TRANSPORT_TCP && netif_running(wg->dev)) + return -EBUSY; list_for_each_entry(peer, &wg->peer_list, peer_list) wg_socket_clear_peer_endpoint_src(peer); if (!netif_running(wg->dev)) { wg->incoming_port = port; return 0; } + return wg_socket_init(wg, port); } static int set_allowedip(struct wg_peer *peer, struct nlattr **attrs) { int ret = -EINVAL; + u32 flags = 0; u16 family; u8 cidr; + wg_dbg("Entering set_allowedip: peer = %px, attrs = %px\n", peer, attrs); + if (!attrs[WGALLOWEDIP_A_FAMILY] || !attrs[WGALLOWEDIP_A_IPADDR] || !attrs[WGALLOWEDIP_A_CIDR_MASK]) return ret; family = nla_get_u16(attrs[WGALLOWEDIP_A_FAMILY]); cidr = nla_get_u8(attrs[WGALLOWEDIP_A_CIDR_MASK]); + if (attrs[WGALLOWEDIP_A_FLAGS]) + flags = nla_get_u32(attrs[WGALLOWEDIP_A_FLAGS]); if (family == AF_INET && cidr <= 32 && - nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr)) - ret = wg_allowedips_insert_v4( - &peer->device->peer_allowedips, - nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, - &peer->device->device_update_lock); - else if (family == AF_INET6 && cidr <= 128 && - nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr)) - ret = wg_allowedips_insert_v6( - &peer->device->peer_allowedips, - nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, - &peer->device->device_update_lock); + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr)) { + if (flags & WGALLOWEDIP_F_REMOVE_ME) + ret = wg_allowedips_remove_v4(&peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + else + ret = wg_allowedips_insert_v4(&peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + } else if (family == AF_INET6 && cidr <= 128 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr)) { + if (flags & WGALLOWEDIP_F_REMOVE_ME) + ret = wg_allowedips_remove_v6(&peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + else + ret = wg_allowedips_insert_v6(&peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + } return ret; } @@ -361,6 +700,8 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) u32 flags = 0; int ret; + wg_dbg("Entering set_peer: wg = %px, attrs = %px\n", wg, attrs); + ret = -EINVAL; if (attrs[WGPEER_A_PUBLIC_KEY] && nla_len(attrs[WGPEER_A_PUBLIC_KEY]) == NOISE_PUBLIC_KEY_LEN) @@ -383,8 +724,7 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) goto out; } - peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, - nla_data(attrs[WGPEER_A_PUBLIC_KEY])); + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, nla_data(attrs[WGPEER_A_PUBLIC_KEY])); ret = 0; if (!peer) { /* Peer doesn't exist yet. Add a new one. */ if (flags & (WGPEER_F_REMOVE_ME | WGPEER_F_UPDATE_ONLY)) @@ -395,9 +735,7 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) down_read(&wg->static_identity.lock); if (wg->static_identity.has_identity && - !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]), - wg->static_identity.static_public, - NOISE_PUBLIC_KEY_LEN)) { + !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]), wg->static_identity.static_public, NOISE_PUBLIC_KEY_LEN)) { /* We silently ignore peers that have the same public * key as the device. The reason we do it silently is * that we'd like for people to be able to reuse the @@ -428,8 +766,7 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) if (preshared_key) { down_write(&peer->handshake.lock); - memcpy(&peer->handshake.preshared_key, preshared_key, - NOISE_SYMMETRIC_KEY_LEN); + memcpy(&peer->handshake.preshared_key, preshared_key, NOISE_SYMMETRIC_KEY_LEN); up_write(&peer->handshake.lock); } @@ -440,24 +777,22 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) if (len == sizeof(struct sockaddr_in) && addr->sa_family == AF_INET) { endpoint.addr4 = *(struct sockaddr_in *)addr; - wg_socket_set_peer_endpoint(peer, &endpoint); + wg_socket_set_peer_endpoint_configured(peer, &endpoint); } else if (len == sizeof(struct sockaddr_in6) && addr->sa_family == AF_INET6) { endpoint.addr6 = *(struct sockaddr_in6 *)addr; - wg_socket_set_peer_endpoint(peer, &endpoint); + wg_socket_set_peer_endpoint_configured(peer, &endpoint); } } if (flags & WGPEER_F_REPLACE_ALLOWEDIPS) - wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer, - &wg->device_update_lock); + wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer, &wg->device_update_lock); if (attrs[WGPEER_A_ALLOWEDIPS]) { struct nlattr *attr, *allowedip[WGALLOWEDIP_A_MAX + 1]; int rem; nla_for_each_nested(attr, attrs[WGPEER_A_ALLOWEDIPS], rem) { - ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX, - attr, allowedip_policy, NULL); + ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX, attr, allowedip_policy, NULL); if (ret < 0) goto out; ret = set_allowedip(peer, allowedip); @@ -467,12 +802,8 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) } if (attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]) { - const u16 persistent_keepalive_interval = nla_get_u16( - attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]); - const bool send_keepalive = - !peer->persistent_keepalive_interval && - persistent_keepalive_interval && - netif_running(wg->dev); + const u16 persistent_keepalive_interval = nla_get_u16(attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]); + const bool send_keepalive = !peer->persistent_keepalive_interval && persistent_keepalive_interval && netif_running(wg->dev); peer->persistent_keepalive_interval = persistent_keepalive_interval; if (send_keepalive) @@ -485,79 +816,115 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) out: wg_peer_put(peer); if (attrs[WGPEER_A_PRESHARED_KEY]) - memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]), - nla_len(attrs[WGPEER_A_PRESHARED_KEY])); + memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]), nla_len(attrs[WGPEER_A_PRESHARED_KEY])); + return ret; } static int wg_set_device(struct sk_buff *skb, struct genl_info *info) { - struct wg_device *wg = lookup_interface(info->attrs, skb); + struct wg_device *wg; u32 flags = 0; int ret; + wg_dbg("Entering wg_set_device: skb = %px, info = %px\n", skb, info); + +#ifdef DIAGNOSTC + /* Decode and print the netlink message received */ + wg_print_netlink_buffer(skb, skb->len); +#endif + + wg = lookup_interface(info->attrs, skb); if (IS_ERR(wg)) { ret = PTR_ERR(wg); + wg_dbg("Error in lookup_interface: %d\n", ret); goto out_nodev; } rtnl_lock(); mutex_lock(&wg->device_update_lock); - if (info->attrs[WGDEVICE_A_FLAGS]) + if (info->attrs[WGDEVICE_A_FLAGS]) { flags = nla_get_u32(info->attrs[WGDEVICE_A_FLAGS]); + } ret = -EOPNOTSUPP; if (flags & ~__WGDEVICE_F_ALL) goto out; - if (info->attrs[WGDEVICE_A_LISTEN_PORT] || info->attrs[WGDEVICE_A_FWMARK]) { + if (info->attrs[WGDEVICE_A_LISTEN_PORT] || + info->attrs[WGDEVICE_A_FWMARK] || + info->attrs[WGDEVICE_A_TRANSPORT]) { struct net *net; rcu_read_lock(); net = rcu_dereference(wg->creating_net); ret = !net || !ns_capable(net->user_ns, CAP_NET_ADMIN) ? -EPERM : 0; rcu_read_unlock(); - if (ret) + if (ret) { + wg_dbg("Permission error for NET_ADMIN capability: %d\n", ret); goto out; + } + } + + if (info->attrs[WGDEVICE_A_TRANSPORT]) { + u8 transport = nla_get_u8(info->attrs[WGDEVICE_A_TRANSPORT]); + + if (transport > WG_TRANSPORT_TCP) { + ret = -EINVAL; + goto out; + } + if (transport != wg->transport) { + if (netif_running(wg->dev) || + (!list_empty(&wg->peer_list) && + !(flags & WGDEVICE_F_REPLACE_PEERS))) { + ret = -EBUSY; + goto out; + } + wg->transport = transport; + } } ++wg->device_update_gen; if (info->attrs[WGDEVICE_A_FWMARK]) { struct wg_peer *peer; + const u32 fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]); + const bool changed = fwmark != wg->fwmark; - wg->fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]); - list_for_each_entry(peer, &wg->peer_list, peer_list) + wg->fwmark = fwmark; + if (wg->transport == WG_TRANSPORT_TCP) + wg_tcp_set_device_mark(wg, fwmark); + list_for_each_entry(peer, &wg->peer_list, peer_list) { wg_socket_clear_peer_endpoint_src(peer); + if (changed && wg->transport == WG_TRANSPORT_TCP) + wg_tcp_peer_request_reconnect(peer); + } } if (info->attrs[WGDEVICE_A_LISTEN_PORT]) { - ret = set_port(wg, - nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT])); + ret = set_port(wg, nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT])); if (ret) goto out; } - if (flags & WGDEVICE_F_REPLACE_PEERS) + if (flags & WGDEVICE_F_REPLACE_PEERS) { wg_peer_remove_all(wg); + } if (info->attrs[WGDEVICE_A_PRIVATE_KEY] && - nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) == - NOISE_PUBLIC_KEY_LEN) { + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) == NOISE_PUBLIC_KEY_LEN) { u8 *private_key = nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]); u8 public_key[NOISE_PUBLIC_KEY_LEN]; struct wg_peer *peer, *temp; bool send_staged_packets; - if (!crypto_memneq(wg->static_identity.static_private, - private_key, NOISE_PUBLIC_KEY_LEN)) + if (!crypto_memneq(wg->static_identity.static_private, private_key, NOISE_PUBLIC_KEY_LEN)) goto skip_set_private_key; /* We remove before setting, to prevent race, which means doing * two 25519-genpub ops. */ if (curve25519_generate_public(public_key, private_key)) { - peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, - public_key); + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, public_key); if (peer) { wg_peer_put(peer); wg_peer_remove(peer); @@ -578,20 +945,23 @@ static int wg_set_device(struct sk_buff *skb, struct genl_info *info) } up_write(&wg->static_identity.lock); } -skip_set_private_key: +skip_set_private_key: if (info->attrs[WGDEVICE_A_PEERS]) { struct nlattr *attr, *peer[WGPEER_A_MAX + 1]; int rem; nla_for_each_nested(attr, info->attrs[WGDEVICE_A_PEERS], rem) { - ret = nla_parse_nested(peer, WGPEER_A_MAX, attr, - peer_policy, NULL); - if (ret < 0) + ret = nla_parse_nested(peer, WGPEER_A_MAX, attr, peer_policy, NULL); + if (ret < 0) { + wg_dbg("Error parsing nested peer attributes: %d\n", ret); goto out; + } ret = set_peer(wg, peer); - if (ret < 0) + if (ret < 0) { + wg_dbg("Error setting peer: %d\n", ret); goto out; + } } } ret = 0; @@ -602,8 +972,10 @@ out: dev_put(wg->dev); out_nodev: if (info->attrs[WGDEVICE_A_PRIVATE_KEY]) - memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]), - nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY])); + memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]), nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY])); + + wg_dbg("Exiting wg_set_device\n"); + return ret; } @@ -614,7 +986,8 @@ static const struct genl_ops genl_ops[] = { .dumpit = wg_get_device_dump, .done = wg_get_device_done, .flags = GENL_UNS_ADMIN_PERM - }, { + }, + { .cmd = WG_CMD_SET_DEVICE, .doit = wg_set_device, .flags = GENL_UNS_ADMIN_PERM @@ -635,10 +1008,20 @@ static struct genl_family genl_family __ro_after_init = { int __init wg_genetlink_init(void) { - return genl_register_family(&genl_family); + wg_dbg("Entering wg_genetlink_init\n"); + + int ret = genl_register_family(&genl_family); + + wg_dbg("Exiting wg_genetlink_init\n"); + + return ret; } void __exit wg_genetlink_uninit(void) { + wg_dbg("Entering wg_genetlink_uninit\n"); + genl_unregister_family(&genl_family); + + wg_dbg("Exiting wg_genetlink_uninit\n"); } diff --git a/kernel/noise.c b/kernel/noise.c index 202a33af5a721f2216ad0815e7b63a28d2bf5888..7b9972c39adf948c07ef2fb8e8db1a1b8b090110 100644 --- a/kernel/noise.c +++ b/kernel/noise.c @@ -641,6 +641,21 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, /* Success! Copy everything to peer */ down_write(&handshake->lock); + /* Resolve simultaneous TCP initiation while the Noise state transition is + * serialized. The lower static public key remains the initiator. + */ + if (wg->transport == WG_TRANSPORT_TCP) { + if (handshake->state == HANDSHAKE_CREATED_INITIATION) { + int cmp = memcmp(wg->static_identity.static_public, + handshake->remote_static, + NOISE_PUBLIC_KEY_LEN); + + if (cmp < 0) { + up_write(&handshake->lock); + goto out; + } + } + } memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0) memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN); diff --git a/kernel/peer.c b/kernel/peer.c index 1cb502a932e07c16ecf91121a294ccafc7005516..d48384eb5a5a92d18927116b305c9744e2619664 100644 --- a/kernel/peer.c +++ b/kernel/peer.c @@ -1,19 +1,24 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ -#include "peer.h" #include "device.h" +#include "noise.h" +#include "peer.h" +#include "peerlookup.h" #include "queueing.h" +#include "socket.h" +#include "wg_tcp.h" #include "timers.h" -#include "peerlookup.h" -#include "noise.h" +#include "wg_tcp_debug.h" #include #include #include #include +#include static struct kmem_cache *peer_cache; static atomic64_t peer_counter = ATOMIC64_INIT(0); @@ -22,19 +27,25 @@ struct wg_peer *wg_peer_create(struct wg_device *wg, const u8 public_key[NOISE_PUBLIC_KEY_LEN], const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) { + wg_dbg("wg_peer_create: entry with wg=%px, public_key=%px, preshared_key=%px\n", wg, public_key, preshared_key); struct wg_peer *peer; int ret = -ENOMEM; lockdep_assert_held(&wg->device_update_lock); - if (wg->num_peers >= MAX_PEERS_PER_DEVICE) + if (wg->num_peers >= MAX_PEERS_PER_DEVICE) { + wg_dbg("wg_peer_create: exit with ERR_PTR(ret)\n"); return ERR_PTR(ret); + } peer = kmem_cache_zalloc(peer_cache, GFP_KERNEL); - if (unlikely(!peer)) + if (unlikely(!peer)) { + wg_dbg("wg_peer_create: exit with ERR_PTR(ret)\n"); return ERR_PTR(ret); - if (unlikely(dst_cache_init(&peer->endpoint_cache, GFP_KERNEL))) - goto err; + } + if (unlikely(dst_cache_init(&peer->endpoint_cache, GFP_KERNEL))) { + goto err_free_peer; + } peer->device = wg; wg_noise_handshake_init(&peer->handshake, &wg->static_identity, @@ -53,32 +64,127 @@ struct wg_peer *wg_peer_create(struct wg_device *wg, kref_init(&peer->refcount); skb_queue_head_init(&peer->staged_packet_queue); wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + /* TCP field initialization */ + peer->peer_socket = NULL; /* Initialize the peer socket to NULL */ + + peer->partial_skb = NULL; /* Initialize the partial skb pointer to NULL */ + peer->expected_len = 0; /* Initialize expected length to 0 */ + peer->received_len = 0; /* Initialize received length to 0 */ + + /* Initialize the TCP retry scheduled flag to false */ + peer->tcp_retry_scheduled = false; + + /* Initialize the delayed work for TCP connection retry */ + INIT_DELAYED_WORK(&peer->tcp_retry_work, wg_tcp_retry_worker); + + /* Initialize the delayed work for TCP socket removal */ + INIT_DELAYED_WORK(&peer->tcp_inbound_remove_work, wg_tcp_inbound_remove_worker); + INIT_DELAYED_WORK(&peer->tcp_outbound_remove_work, wg_tcp_outbound_remove_worker); + + /* Initialize TCP connection status flags */ + peer->tcp_established = false; + peer->tcp_pending = false; + peer->tcp_connecting = false; + peer->tcp_inbound_callbacks_set = false; + peer->tcp_outbound_callbacks_set = false; + peer->tcp_inbound_socket_data = NULL; + peer->tcp_outbound_socket_data = NULL; + peer->tcp_outbound_remove_scheduled = false; + peer->tcp_reconnect_requested = false; + peer->tcp_stopping = false; + peer->tcp_teardown_quarantined = false; + peer->tcp_outbound_remove_socket = NULL; + peer->tcp_inbound_remove_socket = NULL; + peer->tcp_roaming_connection_id = 0; + peer->tcp_inbound_remove_scheduled = false; + peer->peer_endpoint_set = false; + + /* Initialize the spinlock for protecting TCP-related state */ + mutex_init(&peer->tcp_socket_lock); + spin_lock_init(&peer->tcp_lock); + spin_lock_init(&peer->tcp_read_lock); + spin_lock_init(&peer->tcp_write_lock); + + /* Initialize the skb queue for the TX send queue */ + skb_queue_head_init(&peer->send_queue); + + /* Initialize the spinlock for the TX send queue */ + spin_lock_init(&peer->send_queue_lock); + + /* Initialize the list head for pending connection list */ + INIT_LIST_HEAD(&peer->pending_connection_list); + + /* Initialize the work structure, associating it with the worker functions */ + INIT_WORK(&peer->tcp_read_work, wg_tcp_read_worker); + INIT_WORK(&peer->tcp_write_work, wg_tcp_write_worker); + INIT_WORK(&peer->tcp_bootstrap_work, wg_tcp_bootstrap_worker); + peer->tcp_bootstrap_socket = NULL; + INIT_WORK(&peer->tcp_promotion_work, wg_tcp_promotion_worker); + peer->tcp_promotion_connection_id = 0; + peer->tcp_promotion_worker_scheduled = false; + if (wg->transport == WG_TRANSPORT_TCP) { + peer->tcp_read_wq = alloc_workqueue("tcp_read_wq", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!peer->tcp_read_wq) { + pr_err("Failed to allocate read workqueue\n"); + goto err_destroy_endpoint_cache; + } + + peer->tcp_write_wq = alloc_workqueue("tcp_write_wq", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!peer->tcp_write_wq) { + pr_err("Failed to allocate write workqueue\n"); + goto err_destroy_tcp_read_wq; + } + } + + /* Indicate this is a real peer not a temp peer */ + peer->temp_peer = false; + peer->peer_endpoint = peer->endpoint; + set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll); napi_enable(&peer->napi); list_add_tail(&peer->peer_list, &wg->peer_list); INIT_LIST_HEAD(&peer->allowedips_list); wg_pubkey_hashtable_add(wg->peer_hashtable, peer); + ++wg->num_peers; pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); + wg_dbg("wg_peer_create: exit with peer=%px\n", peer); return peer; -err: +err_destroy_tcp_read_wq: + destroy_workqueue(peer->tcp_read_wq); +err_destroy_endpoint_cache: + dst_cache_destroy(&peer->endpoint_cache); +err_free_peer: kmem_cache_free(peer_cache, peer); + wg_dbg("wg_peer_create: exit with ERR_PTR(ret) on err\n"); return ERR_PTR(ret); } struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) { + wg_dbg("wg_peer_get_maybe_zero: entry with peer=%px\n", peer); RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), "Taking peer reference without holding the RCU read lock"); - if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) + if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) { + wg_dbg("wg_peer_get_maybe_zero: exit with NULL\n"); return NULL; + } + wg_dbg("wg_peer_get_maybe_zero: exit with peer=%px\n", peer); return peer; } static void peer_make_dead(struct wg_peer *peer) { + wg_dbg("peer_make_dead: entry with peer=%px\n", peer); + if(!peer || IS_ERR(peer)){ + wg_dbg("Exiting function peer_remove_after_dead, no peer.\n"); + return; + } + /* Remove from configuration-time lookup structures. */ list_del_init(&peer->peer_list); wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, @@ -88,11 +194,57 @@ static void peer_make_dead(struct wg_peer *peer) /* Mark as dead, so that we don't allow jumping contexts after. */ WRITE_ONCE(peer->is_dead, true); + /* Cancel any pending remove/retry delayed work (non-sync to avoid + * self-deadlock if called from a remove worker context). + */ + cancel_delayed_work(&peer->tcp_outbound_remove_work); + peer->tcp_outbound_remove_scheduled = false; + peer->tcp_outbound_remove_socket = NULL; + peer->tcp_inbound_remove_socket = NULL; + peer->tcp_reconnect_requested = false; + peer->tcp_stopping = true; + cancel_delayed_work(&peer->tcp_inbound_remove_work); + peer->tcp_inbound_remove_scheduled = false; + cancel_delayed_work(&peer->tcp_retry_work); + peer->tcp_retry_scheduled = false; + + /* Check if the TCP read work is scheduled before canceling it */ + if (peer->tcp_read_worker_scheduled) { + cancel_work_sync(&peer->tcp_read_work); + peer->tcp_read_worker_scheduled = false; + } + + /* Destroy the TCP read workqueue if it exists */ + if (peer->tcp_read_wq) { + destroy_workqueue(peer->tcp_read_wq); + peer->tcp_read_wq = NULL; /* Avoid dangling pointers */ + } + + /* Check if the TCP write work is scheduled before canceling it */ + if (peer->tcp_write_worker_scheduled) { + cancel_work_sync(&peer->tcp_write_work); + peer->tcp_write_worker_scheduled = false; /* Reset the flag after canceling */ + } + + /* Destroy the TCP write workqueue if it exists */ + if (peer->tcp_write_wq) { + destroy_workqueue(peer->tcp_write_wq); + peer->tcp_write_wq = NULL; /* Avoid dangling pointers */ + } + + /* clean up any partial TCP data if it exists */ + if (peer->partial_skb) { + kfree_skb(peer->partial_skb); + peer->partial_skb = NULL; + } + /* The caller must now synchronize_net() for this to take effect. */ + wg_dbg("peer_make_dead: exit\n"); } static void peer_remove_after_dead(struct wg_peer *peer) { + wg_dbg("peer_remove_after_dead: entry with peer=%px\n", peer); WARN_ON(!peer->is_dead); /* No more keypairs can be created for this peer, since is_dead protects @@ -147,6 +299,7 @@ static void peer_remove_after_dead(struct wg_peer *peer) --peer->device->num_peers; wg_peer_put(peer); + wg_dbg("peer_remove_after_dead: exit\n"); } /* We have a separate "remove" function make sure that all active places where @@ -155,17 +308,25 @@ static void peer_remove_after_dead(struct wg_peer *peer) */ void wg_peer_remove(struct wg_peer *peer) { - if (unlikely(!peer)) + wg_dbg("wg_peer_remove: entry with peer=%px\n", peer); + if (unlikely(!peer)) { + wg_dbg("wg_peer_remove: exit (peer is NULL)\n"); return; + } lockdep_assert_held(&peer->device->device_update_lock); - + /* Claim both directions, detach callbacks, and quiesce stream workers + * before either socket or its sk_user_data wrapper can be released. + */ + wg_tcp_peer_stop(peer); peer_make_dead(peer); synchronize_net(); peer_remove_after_dead(peer); + wg_dbg("wg_peer_remove: exit\n"); } void wg_peer_remove_all(struct wg_device *wg) { + wg_dbg("wg_peer_remove_all: entry with wg=%px\n", wg); struct wg_peer *peer, *temp; LIST_HEAD(dead_peers); @@ -174,6 +335,12 @@ void wg_peer_remove_all(struct wg_device *wg) /* Avoid having to traverse individually for each one. */ wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); + /* First pass: claim and quiesce both TCP directions for every peer before + * peer_make_dead destroys their workqueues. + */ + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_tcp_peer_stop(peer); + list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { peer_make_dead(peer); list_add_tail(&peer->peer_list, &dead_peers); @@ -181,10 +348,12 @@ void wg_peer_remove_all(struct wg_device *wg) synchronize_net(); list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) peer_remove_after_dead(peer); + wg_dbg("wg_peer_remove_all: exit\n"); } static void rcu_release(struct rcu_head *rcu) { + wg_dbg("rcu_release: entry with rcu=%px\n", rcu); struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); dst_cache_destroy(&peer->endpoint_cache); @@ -195,10 +364,12 @@ static void rcu_release(struct rcu_head *rcu) */ memzero_explicit(peer, sizeof(*peer)); kmem_cache_free(peer_cache, peer); + wg_dbg("rcu_release: exit\n"); } static void kref_release(struct kref *refcount) { + wg_dbg("kref_release: entry with refcount=%px\n", refcount); struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", @@ -218,22 +389,31 @@ static void kref_release(struct kref *refcount) /* Free the memory used. */ call_rcu(&peer->rcu, rcu_release); + wg_dbg("kref_release: exit\n"); } void wg_peer_put(struct wg_peer *peer) { - if (unlikely(!peer)) + wg_dbg("wg_peer_put: entry with peer=%px\n", peer); + if (unlikely(!peer)) { + wg_dbg("wg_peer_put: exit (peer is NULL)\n"); return; + } kref_put(&peer->refcount, kref_release); + wg_dbg("wg_peer_put: exit\n"); } int __init wg_peer_init(void) { + wg_dbg("wg_peer_init: entry\n"); peer_cache = KMEM_CACHE(wg_peer, 0); + wg_dbg("wg_peer_init: exit with %d\n", peer_cache ? 0 : -ENOMEM); return peer_cache ? 0 : -ENOMEM; } void wg_peer_uninit(void) { + wg_dbg("wg_peer_uninit: entry\n"); kmem_cache_destroy(peer_cache); + wg_dbg("wg_peer_uninit: exit\n"); } diff --git a/kernel/peer.h b/kernel/peer.h index 76e4d3128ad4ea3f0601cb5055280a4dc097adc2..8d5e81cc45dc001290ee0a1666ed36d2eee2f2ca 100644 --- a/kernel/peer.h +++ b/kernel/peer.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ #ifndef _WG_PEER_H @@ -14,25 +15,11 @@ #include #include #include +#include #include struct wg_device; - -struct endpoint { - union { - struct sockaddr addr; - struct sockaddr_in addr4; - struct sockaddr_in6 addr6; - }; - union { - struct { - struct in_addr src4; - /* Essentially the same as addr6->scope_id */ - int src_if4; - }; - struct in6_addr src6; - }; -}; +struct wg_socket_data; struct wg_peer { struct wg_device *device; @@ -41,7 +28,7 @@ struct wg_peer { int serial_work_cpu; bool is_dead; struct noise_keypairs keypairs; - struct endpoint endpoint; + struct endpoint endpoint, tcp_reply_endpoint, peer_endpoint; struct dst_cache endpoint_cache; rwlock_t endpoint_lock; struct noise_handshake handshake; @@ -64,6 +51,76 @@ struct wg_peer { struct list_head allowedips_list; struct napi_struct napi; u64 internal_id; + u64 tcp_connection_id; /* Nonzero only for a provisional accepted stream. */ + + /* TCP-related members */ + bool peer_endpoint_set; + __be16 tcp_peer_listen_port; /* Configured, never an accepted source port. */ + struct socket *peer_socket, *inbound_socket, *outbound_socket; + struct wg_socket_data *tcp_outbound_socket_data; + struct wg_socket_data *tcp_inbound_socket_data; + /* True while the corresponding socket callbacks are installed. */ + bool tcp_outbound_callbacks_set; + bool tcp_inbound_callbacks_set; + ktime_t outbound_timestamp, inbound_timestamp; /* timestamps for connections */ + struct sockaddr_storage inbound_source, outbound_source, inbound_dest, outbound_dest; + + struct sk_buff *partial_skb; + size_t expected_len; + size_t received_len; + + struct delayed_work tcp_retry_work; /* Work for retrying TCP connection */ + bool tcp_retry_scheduled; /* Flag to track connect retry scheduling */ + + /* Removes the outbound peer TCP connection. */ + struct delayed_work tcp_outbound_remove_work; + bool tcp_outbound_remove_scheduled; /* Flag to track outbound peer removal scheduling */ + bool tcp_reconnect_requested; /* Reconnect after the current outbound socket is quiesced */ + bool tcp_stopping; /* Device/peer stop owns all TCP work cancellation */ + bool tcp_teardown_quarantined; /* Retains callback-reachable state on invariant failure */ + struct socket *tcp_outbound_remove_socket; /* Exact socket claimed by the removal owner */ + u64 tcp_roaming_connection_id; /* Newest authenticated accepted carrier */ + /* Removes the inbound peer TCP connection. */ + struct delayed_work tcp_inbound_remove_work; + bool tcp_inbound_remove_scheduled; /* Flag to track inbound peer removal scheduling */ + struct socket *tcp_inbound_remove_socket; /* Exact socket claimed by the removal owner */ + + /* Removes TCP connections from the pending list. */ + struct delayed_work tcp_cleanup_work; + bool tcp_cleanup_scheduled; /* Flag to track removal scheduling */ + + bool tcp_established; /* Flag to track TCP connection status */ + bool tcp_pending; /* Flag to track outbount pending TCP connection status */ + bool tcp_connecting; /* Synchronous connect setup owns outbound cleanup */ + bool inbound_connected; /* peer connected to us */ + bool outbound_connected; /* we connected to them */ + bool clean_outbound; /* release outbound at next cleanup */ + bool clean_inbound; /* release inbound at next cleanup */ + bool temp_peer; /* is this a temporary peer */ + + + struct sk_buff_head send_queue; /* TX queue */ + spinlock_t send_queue_lock; /* TX lock */ + + struct list_head pending_connection_list; /* peers pending connection handshake */ + struct mutex tcp_socket_lock; /* Serializes socket publication and callback ownership */ + spinlock_t tcp_lock; /* Protects TCP-related state */ + + struct work_struct tcp_read_work; /* Work struct for scheduling the worker */ + struct workqueue_struct *tcp_read_wq; /* Workqueue for handling TCP data processing */ + spinlock_t tcp_read_lock; /* Spinlock to protect access to the socket data */ + bool tcp_read_worker_scheduled; /* Flag to indicate if the TCP read worker is scheduled */ + + struct work_struct tcp_write_work; /* Work struct for scheduling the worker */ + struct workqueue_struct *tcp_write_wq; /* Workqueue for handling TCP data processing */ + spinlock_t tcp_write_lock; /* Spinlock to protect access to the socket data */ + bool tcp_write_worker_scheduled; /* Flag to indicate if the TCP write worker is scheduled */ + struct work_struct tcp_bootstrap_work; + struct socket *tcp_bootstrap_socket; + struct work_struct tcp_promotion_work; + u64 tcp_promotion_connection_id; + bool tcp_promotion_worker_scheduled; + }; struct wg_peer *wg_peer_create(struct wg_device *wg, @@ -76,6 +133,7 @@ static inline struct wg_peer *wg_peer_get(struct wg_peer *peer) kref_get(&peer->refcount); return peer; } + void wg_peer_put(struct wg_peer *peer); void wg_peer_remove(struct wg_peer *peer); void wg_peer_remove_all(struct wg_device *wg); diff --git a/kernel/queueing.h b/kernel/queueing.h index 1ea4f874e367ee4185efd0cce9f2358c0d3d5e01..5b2493fe94f08ac1d36085a28a39db38f9895093 100644 --- a/kernel/queueing.h +++ b/kernel/queueing.h @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include "wg_tcp_debug.h" struct wg_device; struct wg_peer; @@ -58,10 +60,14 @@ enum packet_state { struct packet_cb { u64 nonce; + u64 tcp_connection_id; struct noise_keypair *keypair; atomic_t state; u32 mtu; + __be16 frag_id; + __be16 frag_off; u8 ds; + u8 outer_ipproto; }; #define PACKET_CB(skb) ((struct packet_cb *)((skb)->cb)) @@ -75,6 +81,7 @@ static inline bool wg_check_packet_protocol(struct sk_buff *skb) static inline void wg_reset_packet(struct sk_buff *skb, bool encapsulating) { + wg_dbg("Entering wg_reset_packet\n"); u8 l4_hash = skb->l4_hash; u8 sw_hash = skb->sw_hash; u32 hash = skb->hash; @@ -100,6 +107,8 @@ static inline void wg_reset_packet(struct sk_buff *skb, bool encapsulating) skb_reset_transport_header(skb); skb_probe_transport_header(skb); skb_reset_inner_headers(skb); + + wg_dbg("Exiting wg_reset_packet\n"); } static inline int wg_cpumask_choose_online(int *stored_cpu, unsigned int id) @@ -159,7 +168,7 @@ static inline int wg_queue_enqueue_per_device_and_peer( struct sk_buff *skb, struct workqueue_struct *wq) { int cpu; - + wg_dbg("Entering wg_queue_enqueue_per_device_and_peer \n"); atomic_set_release(&PACKET_CB(skb)->state, PACKET_STATE_UNCRYPTED); /* We first queue this up for the peer ingestion, but the consumer * will wait for the state to change to CRYPTED or DEAD before. @@ -174,11 +183,13 @@ static inline int wg_queue_enqueue_per_device_and_peer( if (unlikely(ptr_ring_produce_bh(&device_queue->ring, skb))) return -EPIPE; queue_work_on(cpu, wq, &per_cpu_ptr(device_queue->worker, cpu)->work); + wg_dbg("Exiting wg_queue_enqueue_per_device_and_peer \n"); return 0; } static inline void wg_queue_enqueue_per_peer_tx(struct sk_buff *skb, enum packet_state state) { + wg_dbg("Entering wg_queue_enqueue_per_peer_tx \n"); /* We take a reference, because as soon as we call atomic_set, the * peer can be freed from below us. */ @@ -188,10 +199,12 @@ static inline void wg_queue_enqueue_per_peer_tx(struct sk_buff *skb, enum packet queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, peer->internal_id), peer->device->packet_crypt_wq, &peer->transmit_packet_work); wg_peer_put(peer); + wg_dbg("Exiting wg_queue_enqueue_per_peer_tx\n"); } static inline void wg_queue_enqueue_per_peer_rx(struct sk_buff *skb, enum packet_state state) { + wg_dbg("Entering wg_queue_enqueue_per_peer_rx \n"); /* We take a reference, because as soon as we call atomic_set, the * peer can be freed from below us. */ @@ -200,6 +213,7 @@ static inline void wg_queue_enqueue_per_peer_rx(struct sk_buff *skb, enum packet atomic_set_release(&PACKET_CB(skb)->state, state); napi_schedule(&peer->napi); wg_peer_put(peer); + wg_dbg("Exiting wg_queue_enqueue_per_peer_rx\n"); } #ifdef DEBUG diff --git a/kernel/receive.c b/kernel/receive.c index a176653c88616b1bc871fe52fcea778b5e189f69..a4e5c83d5cb77f520bba42113f1cd6eaca2c3d2d 100644 --- a/kernel/receive.c +++ b/kernel/receive.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ +#include "allowedips.h" #include "queueing.h" #include "device.h" #include "peer.h" @@ -10,88 +12,243 @@ #include "messages.h" #include "cookie.h" #include "socket.h" +#include "wg_tcp.h" +#include "wg_tcp_debug.h" +#include #include #include +#include +#include +#include +#include #include #include +#include +#include /* Must be called with bh disabled. */ static void update_rx_stats(struct wg_peer *peer, size_t len) { + wg_dbg("Entering update_rx_stats: peer=%px, len=%zu\n", peer, len); dev_sw_netstats_rx_add(peer->device->dev, len); peer->rx_bytes += len; + wg_dbg("Exiting update_rx_stats\n"); } #define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) static size_t validate_header_len(struct sk_buff *skb) { - if (unlikely(skb->len < sizeof(struct message_header))) + wg_dbg("Entering validate_header_len: skb=%px\n", skb); + wg_dbg("SKB state: len=%d, head=%px, data=%px, tail=%u, end=%u\n", + skb->len, skb->head, skb->data, skb->tail, skb->end); + + wg_dbg("sizeof(struct message_header)=%zu\n", sizeof(struct message_header)); + if (unlikely(skb->len < sizeof(struct message_header))) { + wg_dbg("Exiting validate_header_len: skb len (%d) is less " + "than sizeof(struct message_header) (%zu)\n", + skb->len, (size_t)sizeof(struct message_header)); return 0; - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && - skb->len >= MESSAGE_MINIMUM_LENGTH) - return sizeof(struct message_data); - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && - skb->len == sizeof(struct message_handshake_initiation)) - return sizeof(struct message_handshake_initiation); - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && - skb->len == sizeof(struct message_handshake_response)) - return sizeof(struct message_handshake_response); - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && - skb->len == sizeof(struct message_handshake_cookie)) - return sizeof(struct message_handshake_cookie); + } + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA)) { + wg_dbg("SKB_TYPE_LE32(skb) matches MESSAGE_DATA, checking length.\n"); + wg_dbg("MESSAGE_MINIMUM_LENGTH=%d, sizeof(struct message_data)=%zu\n", + MESSAGE_MINIMUM_LENGTH, sizeof(struct message_data)); + if (skb->len >= MESSAGE_MINIMUM_LENGTH) { + wg_dbg("Exiting validate_header_len: skb len (%d) is greater than or " + "equal to MESSAGE_MINIMUM_LENGTH (%d), returning sizeof(struct " + "message_data) (%zu)\n", + skb->len, MESSAGE_MINIMUM_LENGTH, sizeof(struct message_data)); + return sizeof(struct message_data); + } + } + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION)) { + wg_dbg("SKB_TYPE_LE32(skb) matches MESSAGE_HANDSHAKE_INITIATION, checking length.\n"); + wg_dbg("sizeof(struct message_handshake_initiation)=%zu\n", + sizeof(struct message_handshake_initiation)); + if (skb->len == sizeof(struct message_handshake_initiation)) { + wg_dbg("Exiting validate_header_len: skb len (%d) matches " + "sizeof(struct message_handshake_initiation) (%zu)\n", + skb->len, sizeof(struct message_handshake_initiation)); + return sizeof(struct message_handshake_initiation); + } + } + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE)) { + wg_dbg("SKB_TYPE_LE32(skb) matches MESSAGE_HANDSHAKE_RESPONSE, checking length.\n"); + wg_dbg("sizeof(struct message_handshake_response)=%zu\n", + sizeof(struct message_handshake_response)); + if (skb->len == sizeof(struct message_handshake_response)) { + wg_dbg("Exiting validate_header_len: skb len (%d) matches " + "sizeof(struct message_handshake_response) (%zu)\n", + skb->len, sizeof(struct message_handshake_response)); + return sizeof(struct message_handshake_response); + } + } + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { + wg_dbg("SKB_TYPE_LE32(skb) matches MESSAGE_HANDSHAKE_COOKIE, checking length.\n"); + wg_dbg("sizeof(struct message_handshake_cookie)=%zu\n", + sizeof(struct message_handshake_cookie)); + if (skb->len == sizeof(struct message_handshake_cookie)) { + wg_dbg("Exiting validate_header_len: skb len (%d) matches " + "sizeof(struct message_handshake_cookie) (%zu)\n", + skb->len, sizeof(struct message_handshake_cookie)); + return sizeof(struct message_handshake_cookie); + } + } + + wg_dbg("Exiting validate_header_len: no valid message type found or length mismatch.\n"); return 0; } static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) { + wg_dbg("Entering prepare_skb_header: skb=%px, wg=%px\n", skb, wg); size_t data_offset, data_len, header_len; - struct udphdr *udp; - + struct udphdr _udp, *udp; + wg_dbg("wg: prepare_skb_header: ENTER\n" + " skb->len=%u, skb->data_len=%u\n" + " headroom=%u, tailroom=%u\n" + " skb->head=%px, skb->data=%px, skb->tail=%u, skb->end=%u\n", + skb->len, skb->data_len, + skb_headroom(skb), skb_tailroom(skb), + skb->head, skb->data, skb->tail, skb->end); + + wg_dbg("Initial skb state: head=%px, data=%px, tail=%u, end=%u, len=%d, headroom=%d\n", + skb->head, skb->data, skb->tail, skb->end, skb->len, skb_headroom(skb)); + + /* Check packet protocol and header validity */ if (unlikely(!wg_check_packet_protocol(skb) || skb_transport_header(skb) < skb->head || (skb_transport_header(skb) + sizeof(struct udphdr)) > - skb_tail_pointer(skb))) + skb_tail_pointer(skb))) { + wg_dbg("Exiting prepare_skb_header with error -EINVAL: " + "Invalid transport header or protocol check failed.\n"); return -EINVAL; /* Bogus IP header */ - udp = udp_hdr(skb); - data_offset = (u8 *)udp - skb->data; + } + wg_dbg("wg: prepare_skb_header: protocol checks " + "passed.\n"); + + /* Safely access UDP header using skb_header_pointer */ + udp = skb_header_pointer(skb, skb_transport_offset(skb), sizeof(_udp), &_udp); + if (!udp) { + wg_dbg("Exiting prepare_skb_header with error " + "-EINVAL: Failed to access UDP header using " + "skb_header_pointer.\n"); + return -EINVAL; + } + + wg_dbg("UDP header source=%u, dest=%u\n", + ntohs(udp->source), ntohs(udp->dest)); + + /* Calculate data offset and validate */ + data_offset = skb_transport_offset(skb) + sizeof(struct udphdr); + wg_dbg("Data offset calculated: data_offset=%zu\n", + data_offset); + if (unlikely(data_offset > U16_MAX || - data_offset + sizeof(struct udphdr) > skb->len)) - /* Packet has offset at impossible location or isn't big enough - * to have UDP fields. - */ + data_offset + sizeof(struct udphdr) > skb->len)) { + wg_dbg("Exiting prepare_skb_header with error" + " -EINVAL: Invalid data offset or UDP header size " + "too large.\n"); return -EINVAL; + } + + /* Get the UDP length field */ data_len = ntohs(udp->len); + wg_dbg("UDP length field: data_len=%zu\n", data_len); + + /* Validate data length */ if (unlikely(data_len < sizeof(struct udphdr) || - data_len > skb->len - data_offset)) - /* UDP packet is reporting too small of a size or lying about - * its size. - */ + data_len > skb->len - skb_transport_offset(skb))) { + wg_dbg("Exiting prepare_skb_header with error " + "-EINVAL: UDP length field too small or larger than " + "available data.\n"); + return -EINVAL; + } + + /* Adjust data length to exclude UDP header */ data_len -= sizeof(struct udphdr); - data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; + data_offset = skb_transport_offset(skb) + sizeof(struct udphdr); + wg_dbg("Adjusted data_len=%zu, adjusted data_offset=%zu\n", + data_len, data_offset); + + /* Check pull and trim capabilities */ if (unlikely(!pskb_may_pull(skb, data_offset + sizeof(struct message_header)) || - pskb_trim(skb, data_len + data_offset) < 0)) + pskb_trim(skb, data_len + data_offset) < 0)) { + wg_dbg("Exiting prepare_skb_header with error " + "-EINVAL: pskb_may_pull or pskb_trim failed. " + "data_offset=%zu, data_len=%zu, len=%d\n", + data_offset, data_len, skb->len); return -EINVAL; + } + + /* Diagnostics before pulling SKB data */ + wg_dbg("Before skb_pull: len=%d, data=%px, tail=%u\n", skb->len, + skb->data, skb->tail); skb_pull(skb, data_offset); - if (unlikely(skb->len != data_len)) - /* Final len does not agree with calculated len */ + /* Diagnostics after pulling SKB data */ + wg_dbg("After skb_pull: len=%d, data=%px, tail=%u\n", skb->len, + skb->data, skb->tail); + + /* Validate the SKB length against calculated data length */ + if (unlikely(skb->len != data_len)) { + wg_dbg("Exiting prepare_skb_header with error -EINVAL: " + "Final length does not match calculated length. len=%d, expected data_len=%zu\n", + skb->len, data_len); return -EINVAL; + } + + /* Validate header length */ header_len = validate_header_len(skb); - if (unlikely(!header_len)) + wg_dbg("Header length validated: header_len=%zu\n", header_len); + + if (unlikely(!header_len)) { + wg_dbg("Exiting prepare_skb_header with error -EINVAL\n"); return -EINVAL; + } + + /* Diagnostics before pushing SKB data back */ + wg_dbg("Before __skb_push: len=%d, data=%px, tail=%u\n", skb->len, + skb->data, skb->tail); __skb_push(skb, data_offset); - if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) + /* Diagnostics after pushing SKB data back */ + wg_dbg("After __skb_push: len=%d, data=%px, tail=%u\n", skb->len, + skb->data, skb->tail); + + /* Check pull capabilities after push */ + if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) { + wg_dbg("Exiting prepare_skb_header with error -EINVAL: " + "pskb_may_pull failed after __skb_push. data_offset=%zu, header_len=%zu\n", + data_offset, header_len); return -EINVAL; + } + + /* Diagnostics before pulling SKB data again */ + wg_dbg("Before __skb_pull: len=%d, data=%px, tail=%u\n", skb->len, + skb->data, skb->tail); __skb_pull(skb, data_offset); + /* Diagnostics after pulling SKB data again */ + wg_dbg("After __skb_pull: len=%d, data=%px, tail=%u\n", skb->len, + skb->data, skb->tail); + + wg_dbg("Exiting prepare_skb_header successfully: " + "final len=%d, data=%px, head=%px, tail=%u, end=%u, headroom=%d, " + "tailroom=%d\n", skb->len, skb->data, skb->head, skb->tail, + skb->end, skb_headroom(skb), skb_tailroom(skb)); return 0; } static void wg_receive_handshake_packet(struct wg_device *wg, struct sk_buff *skb) { + wg_dbg("Entering wg_receive_handshake_packet: wg=%px, skb=%px\n", wg, skb); enum cookie_mac_state mac_state; struct wg_peer *peer = NULL; /* This is global, so that our load calculation applies to the whole @@ -100,100 +257,130 @@ static void wg_receive_handshake_packet(struct wg_device *wg, static u64 last_under_load; bool packet_needs_cookie; bool under_load; + enum cookie_validation_action cookie_action; + + wg_dbg("Validating handshake packet with len=%u\n", skb->len); + wg_dbg("Received Handshake Packet: %*ph\n", (int)skb->len, skb->data); + + if(wg->transport == WG_TRANSPORT_TCP) { + /* For TCP, skip cookie check */ + packet_needs_cookie = false; + goto nocookie; + } + /* Handle handshake cookie response */ if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { - net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", - wg->dev->name, skb); - wg_cookie_message_consume( - (struct message_handshake_cookie *)skb->data, wg); + net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", wg->dev->name, skb); + wg_cookie_message_consume((struct message_handshake_cookie *)skb->data, wg); + wg_dbg("Exiting wg_receive_handshake_packet\n"); return; } - under_load = atomic_read(&wg->handshake_queue_len) >= - MAX_QUEUED_INCOMING_HANDSHAKES / 8; + /* Load calculation to decide if system is under load */ + under_load = atomic_read(&wg->handshake_queue_len) >= MAX_QUEUED_INCOMING_HANDSHAKES / 8; if (under_load) { last_under_load = ktime_get_coarse_boottime_ns(); + wg_dbg("System under load: last_under_load set to %llu\n", last_under_load); } else if (last_under_load) { under_load = !wg_birthdate_has_expired(last_under_load, 1); - if (!under_load) + if (!under_load) { last_under_load = 0; + wg_dbg("System load normalized: last_under_load reset\n"); + } } - mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, - under_load); - if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || - (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { + + /* Validate packet's MAC and set packet_needs_cookie flag */ + mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, under_load); + wg_dbg("MAC validation result: %d\n", mac_state); + cookie_action = wg_cookie_validation_action(under_load, mac_state); + if (cookie_action == WG_COOKIE_ACCEPT) { packet_needs_cookie = false; - } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { + } else if (cookie_action == WG_COOKIE_CHALLENGE) { packet_needs_cookie = true; } else { - net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", - wg->dev->name, skb); + net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", wg->dev->name, skb); + wg_dbg("Exiting wg_receive_handshake_packet\n"); return; } +nocookie: + /* Process handshake packets */ switch (SKB_TYPE_LE32(skb)) { case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { - struct message_handshake_initiation *message = - (struct message_handshake_initiation *)skb->data; + struct message_handshake_initiation *message = (struct message_handshake_initiation *)skb->data; + wg_dbg("Processing handshake initiation packet\n"); if (packet_needs_cookie) { - wg_packet_send_handshake_cookie(wg, skb, - message->sender_index); + wg_packet_send_handshake_cookie(wg, skb, message->sender_index); + wg_dbg("Exiting wg_receive_handshake_packet: Cookie sent for initiation\n"); return; } + + /* Handle handshake initiation */ peer = wg_noise_handshake_consume_initiation(message, wg); if (unlikely(!peer)) { - net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", - wg->dev->name, skb); + net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", wg->dev->name, skb); + wg_dbg("Exiting wg_receive_handshake_packet\n"); return; } - wg_socket_set_peer_endpoint_from_skb(peer, skb); - net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", - wg->dev->name, peer->internal_id, - &peer->endpoint.addr); + print_peer_socket_info(peer); + if (wg->transport == WG_TRANSPORT_UDP) + wg_socket_set_peer_endpoint_from_skb(peer, skb); + else if (PACKET_CB(skb)->outer_ipproto == IPPROTO_TCP) + wg_socket_set_peer_endpoint_authenticated_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", wg->dev->name, peer->internal_id, &peer->endpoint.addr); wg_packet_send_handshake_response(peer); break; } case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { - struct message_handshake_response *message = - (struct message_handshake_response *)skb->data; + struct message_handshake_response *message = (struct message_handshake_response *)skb->data; + wg_dbg("Processing handshake response packet\n"); if (packet_needs_cookie) { - wg_packet_send_handshake_cookie(wg, skb, - message->sender_index); + wg_packet_send_handshake_cookie(wg, skb, message->sender_index); + wg_dbg("Exiting wg_receive_handshake_packet: Cookie sent for response\n"); return; } + + /* Handle handshake response */ peer = wg_noise_handshake_consume_response(message, wg); if (unlikely(!peer)) { - net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", - wg->dev->name, skb); + wg_dbg("Peer object is NULL. Dropping packet.\n"); + net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", wg->dev->name, skb); + wg_dbg("Exiting wg_receive_handshake_packet\n"); return; } - wg_socket_set_peer_endpoint_from_skb(peer, skb); - net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", - wg->dev->name, peer->internal_id, - &peer->endpoint.addr); - if (wg_noise_handshake_begin_session(&peer->handshake, - &peer->keypairs)) { + + print_peer_socket_info(peer); + if (peer->device->transport == WG_TRANSPORT_UDP) { + wg_socket_set_peer_endpoint_from_skb(peer, skb); + } else if (PACKET_CB(skb)->outer_ipproto == IPPROTO_TCP) { + wg_socket_set_peer_endpoint_authenticated_from_skb(peer, skb); + } + net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", wg->dev->name, peer->internal_id, &peer->endpoint.addr); + + if (wg_noise_handshake_begin_session(&peer->handshake, &peer->keypairs)) { wg_timers_session_derived(peer); wg_timers_handshake_complete(peer); - /* Calling this function will either send any existing - * packets in the queue and not send a keepalive, which - * is the best case, Or, if there's nothing in the - * queue, it will send a keepalive, in order to give - * immediate confirmation of the session. - */ wg_packet_send_keepalive(peer); } break; } + + default: + wg_dbg("Unknown packet type received in handshake processing: %u\n", + SKB_TYPE_LE32(skb)); + break; } + /* Final check to ensure peer is valid */ if (unlikely(!peer)) { - WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); + WARN(1, "Unexpected state: No valid peer found after handshake processing\n"); + wg_dbg("Exiting wg_receive_handshake_packet\n"); return; } + /* Update statistics and state */ local_bh_disable(); update_rx_stats(peer, skb->len); local_bh_enable(); @@ -201,10 +388,12 @@ static void wg_receive_handshake_packet(struct wg_device *wg, wg_timers_any_authenticated_packet_received(peer); wg_timers_any_authenticated_packet_traversal(peer); wg_peer_put(peer); + wg_dbg("Exiting wg_receive_handshake_packet\n"); } void wg_packet_handshake_receive_worker(struct work_struct *work) { + wg_dbg("Entering wg_packet_handshake_receive_worker: work=%px\n", work); struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr; struct wg_device *wg = container_of(queue, struct wg_device, handshake_queue); struct sk_buff *skb; @@ -215,10 +404,12 @@ void wg_packet_handshake_receive_worker(struct work_struct *work) atomic_dec(&wg->handshake_queue_len); cond_resched(); } + wg_dbg("Exiting wg_packet_handshake_receive_worker\n"); } static void keep_key_fresh(struct wg_peer *peer) { + wg_dbg("Entering keep_key_fresh: peer=%px\n", peer); struct noise_keypair *keypair; bool send; @@ -237,6 +428,7 @@ static void keep_key_fresh(struct wg_peer *peer) peer->sent_lastminute_handshake = true; wg_packet_send_queued_handshake_initiation(peer, false); } + wg_dbg("Exiting keep_key_fresh\n"); } static bool decrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) @@ -246,54 +438,94 @@ static bool decrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) unsigned int offset; int num_frags; - if (unlikely(!keypair)) + wg_dbg("Entering decrypt_packet: skb=%px, keypair=%px\n", skb, keypair); + wg_dbg("skb->len = %u, skb->data_len = %u, skb->network_header = %px\n", + skb->len, skb->data_len, skb_network_header(skb)); + + if (unlikely(!keypair)) { + wg_dbg("Keypair is NULL\n"); + wg_dbg("Exiting decrypt_packet with false\n"); return false; + } if (unlikely(!READ_ONCE(keypair->receiving.is_valid) || - wg_birthdate_has_expired(keypair->receiving.birthdate, REJECT_AFTER_TIME) || - keypair->receiving_counter.counter >= REJECT_AFTER_MESSAGES)) { + wg_birthdate_has_expired(keypair->receiving.birthdate, REJECT_AFTER_TIME) || + READ_ONCE(keypair->receiving_counter.counter) >= REJECT_AFTER_MESSAGES)) { WRITE_ONCE(keypair->receiving.is_valid, false); + wg_dbg("Keypair is invalid or expired: is_valid=%d, counter=%llu\n", + keypair->receiving.is_valid, keypair->receiving_counter.counter); + wg_dbg("Exiting decrypt_packet with false\n"); return false; } - PACKET_CB(skb)->nonce = - le64_to_cpu(((struct message_data *)skb->data)->counter); + PACKET_CB(skb)->nonce = le64_to_cpu(((struct message_data *)skb->data)->counter); + wg_dbg("Extracted nonce from skb: nonce=%llu\n", PACKET_CB(skb)->nonce); + wg_dbg("skb->data (before decryption): %*ph\n", skb->len, skb->data); - /* We ensure that the network header is part of the packet before we - * call skb_cow_data, so that there's no chance that data is removed - * from the skb, so that later we can extract the original endpoint. - */ + /* Ensure network header is part of the packet */ offset = skb->data - skb_network_header(skb); + wg_dbg("Pushing skb to preserve network header, offset=%u\n", offset); skb_push(skb, offset); num_frags = skb_cow_data(skb, 0, &trailer); + wg_dbg("num_frags after skb_cow_data: %d\n", num_frags); offset += sizeof(struct message_data); skb_pull(skb, offset); - if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + wg_dbg("Pulled skb to offset: %u, skb->len=%u, skb->data=%px\n", offset, skb->len, skb->data); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) { + wg_dbg("skb->data (after decryption failed): %*ph\n", skb->len, skb->data); + wg_dbg("Failed skb_cow_data: num_frags=%d, skb->len=%u\n", num_frags, skb->len); + wg_dbg("Exiting decrypt_packet with false\n"); return false; + } sg_init_table(sg, num_frags); - if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) + if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) { + wg_dbg("Failed skb_to_sgvec, skb->len=%u\n", skb->len); + wg_dbg("Exiting decrypt_packet with false\n"); return false; + } + + wg_dbg("Scattergather segments prepared, starting decryption\n"); +#define NOISE_KEY_LEN 32 + wg_dbg("Decryption key: %*ph\n", NOISE_KEY_LEN, keypair->receiving.key); if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, - PACKET_CB(skb)->nonce, - keypair->receiving.key)) - return false; + PACKET_CB(skb)->nonce, + keypair->receiving.key)) { + wg_dbg("skb->data (after decryption failed): %*ph\n", + skb->len, skb->data); + wg_dbg("Decryption failed\n"); + wg_dbg("Exiting decrypt_packet with false\n"); + return false; + } +#ifdef WG_TCP_VERBOSE + decode_and_print_packet(skb, "[decrypt]"); +#endif - /* Another ugly situation of pushing and pulling the header so as to - * keep endpoint information intact. - */ + /* Ensure endpoint information remains intact */ + wg_dbg("Pushing skb to preserve endpoint information\n"); skb_push(skb, offset); - if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) + if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) { + wg_dbg("skb->data (after decryption failed): %*ph\n", skb->len, skb->data); + wg_dbg("Failed pskb_trim, skb->len=%u\n", skb->len); + wg_dbg("Exiting decrypt_packet with false\n"); return false; + } skb_pull(skb, offset); + wg_dbg("skb->data (after decryption succeeded): %*ph\n", + skb->len, skb->data); + wg_dbg("Pulled skb to offset: %u, skb->len=%u, skb->data=%px\n", + offset, skb->len, skb->data); + + wg_dbg("Exiting decrypt_packet with true, skb->len=%u\n", skb->len); return true; } /* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ static bool counter_validate(struct noise_replay_counter *counter, u64 their_counter) { + wg_dbg("Entering counter_validate: counter=%px, their_counter=%llu\n", counter, their_counter); unsigned long index, index_current, top, i; bool ret = false; @@ -318,7 +550,7 @@ static bool counter_validate(struct noise_replay_counter *counter, u64 their_cou for (i = 1; i <= top; ++i) counter->backtrack[(i + index_current) & ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; - counter->counter = their_counter; + WRITE_ONCE(counter->counter, their_counter); } index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; @@ -327,20 +559,36 @@ static bool counter_validate(struct noise_replay_counter *counter, u64 their_cou out: spin_unlock_bh(&counter->lock); + wg_dbg("Exiting counter_validate with %d\n", ret); return ret; } #include "selftest/counter.c" +#include "selftest/cookie.c" static void wg_packet_consume_data_done(struct wg_peer *peer, struct sk_buff *skb, - struct endpoint *endpoint) + struct endpoint *endpoint, + bool authenticated_over_tcp) { + wg_dbg("Entering wg_packet_consume_data_done: peer=%px, skb=%px, endpoint=%px\n", + peer, skb, endpoint); struct net_device *dev = peer->device->dev; unsigned int len, len_before_trim; struct wg_peer *routed_peer; - wg_socket_set_peer_endpoint(peer, endpoint); + if (unlikely(!endpoint)) { + wg_dbg("Endpoint object is NULL. Cannot set peer endpoint.\n"); + return; + } + + + if (peer->device->transport == WG_TRANSPORT_TCP && + authenticated_over_tcp) + wg_socket_set_peer_endpoint_authenticated( + peer, endpoint, PACKET_CB(skb)->tcp_connection_id); + else + wg_socket_set_peer_endpoint(peer, endpoint); if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, PACKET_CB(skb)->keypair))) { @@ -398,6 +646,15 @@ static void wg_packet_consume_data_done(struct wg_peer *peer, if (unlikely(len > skb->len)) goto dishonest_packet_size; len_before_trim = skb->len; + + + if (unlikely(len == 0 || len_before_trim == 0)) { + wg_dbg("Invalid packet length detected: len=%u, len_before_trim=%u\n", + len, len_before_trim); + return; + } + + if (unlikely(pskb_trim(skb, len))) goto packet_processed; @@ -410,6 +667,7 @@ static void wg_packet_consume_data_done(struct wg_peer *peer, napi_gro_receive(&peer->napi, skb); update_rx_stats(peer, message_data_len(len_before_trim)); + wg_dbg("Exiting wg_packet_consume_data_done\n"); return; dishonest_packet_peer: @@ -433,10 +691,12 @@ dishonest_packet_size: goto packet_processed; packet_processed: dev_kfree_skb(skb); + wg_dbg("Exiting wg_packet_consume_data_done\n"); } int wg_packet_rx_poll(struct napi_struct *napi, int budget) { + wg_dbg("Entering wg_packet_rx_poll: napi=%px, budget=%d\n", napi, budget); struct wg_peer *peer = container_of(napi, struct wg_peer, napi); struct noise_keypair *keypair; struct endpoint endpoint; @@ -444,9 +704,12 @@ int wg_packet_rx_poll(struct napi_struct *napi, int budget) struct sk_buff *skb; int work_done = 0; bool free; + bool authenticated_over_tcp; - if (unlikely(budget <= 0)) + if (unlikely(budget <= 0)) { + wg_dbg("Exiting wg_packet_rx_poll with 0\n"); return 0; + } while ((skb = wg_prev_queue_peek(&peer->rx_queue)) != NULL && (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != @@ -463,15 +726,18 @@ int wg_packet_rx_poll(struct napi_struct *napi, int budget) net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", peer->device->dev->name, PACKET_CB(skb)->nonce, - keypair->receiving_counter.counter); + READ_ONCE(keypair->receiving_counter.counter)); goto next; } if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) goto next; + authenticated_over_tcp = + PACKET_CB(skb)->outer_ipproto == IPPROTO_TCP; wg_reset_packet(skb, false); - wg_packet_consume_data_done(peer, skb, &endpoint); + wg_packet_consume_data_done(peer, skb, &endpoint, + authenticated_over_tcp); free = false; next: @@ -487,11 +753,13 @@ next: if (work_done < budget) napi_complete_done(napi, work_done); + wg_dbg("Exiting wg_packet_rx_poll with %d\n", work_done); return work_done; } void wg_packet_decrypt_worker(struct work_struct *work) { + wg_dbg("Entering wg_packet_decrypt_worker: work=%px\n", work); struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr; struct sk_buff *skb; @@ -504,14 +772,19 @@ void wg_packet_decrypt_worker(struct work_struct *work) if (need_resched()) cond_resched(); } + wg_dbg("Exiting wg_packet_decrypt_worker\n"); } static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) { + wg_dbg("Entering wg_packet_consume_data: wg=%px, skb=%px\n", wg, skb); __le32 idx = ((struct message_data *)skb->data)->key_idx; struct wg_peer *peer = NULL; int ret; + wg_dbg("Consuming data packet with key_idx=%u\n", idx); + wg_dbg("Data Packet Contents: %*ph\n", (int)skb->len, skb->data); + rcu_read_lock_bh(); PACKET_CB(skb)->keypair = (struct noise_keypair *)wg_index_hashtable_lookup( @@ -529,6 +802,7 @@ static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) wg_queue_enqueue_per_peer_rx(skb, PACKET_STATE_DEAD); if (likely(!ret || ret == -EPIPE)) { rcu_read_unlock_bh(); + wg_dbg("Exiting wg_packet_consume_data\n"); return; } err: @@ -537,50 +811,86 @@ err_keypair: rcu_read_unlock_bh(); wg_peer_put(peer); dev_kfree_skb(skb); + wg_dbg("Exiting wg_packet_consume_data\n"); } void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) { - if (unlikely(prepare_skb_header(skb, wg) < 0)) + wg_dbg("Entering wg_packet_receive: wg=%px, skb=%px\n", wg, skb); + + if (unlikely(prepare_skb_header(skb, wg) < 0)) { + wg_dbg("prepare_skb_header failed\n"); goto err; - switch (SKB_TYPE_LE32(skb)) { + } + + /* Determine packet type */ + uint32_t skb_type = SKB_TYPE_LE32(skb); + wg_dbg("Packet type: %u\n", skb_type); + + switch (skb_type) { case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { int cpu, ret = -EBUSY; + wg_dbg("Received handshake packet\n"); - if (unlikely(!rng_is_initialized())) + if (unlikely(!rng_is_initialized())) { + wg_dbg("RNG is not initialized, dropping packet\n"); goto drop; - if (atomic_read(&wg->handshake_queue_len) > MAX_QUEUED_INCOMING_HANDSHAKES / 2) { + } + + int queue_len = atomic_read(&wg->handshake_queue_len); + wg_dbg("Current handshake queue length: %d\n", queue_len); + + if (queue_len > MAX_QUEUED_INCOMING_HANDSHAKES / 2) { + wg_dbg("Queue length exceeds threshold, trying spinlock\n"); if (spin_trylock_bh(&wg->handshake_queue.ring.producer_lock)) { ret = __ptr_ring_produce(&wg->handshake_queue.ring, skb); + wg_dbg("__ptr_ring_produce returned: %d\n", ret); spin_unlock_bh(&wg->handshake_queue.ring.producer_lock); + } else { + wg_dbg("Failed to acquire spinlock\n"); } - } else + } else { ret = ptr_ring_produce_bh(&wg->handshake_queue.ring, skb); + wg_dbg("ptr_ring_produce_bh returned: %d\n", ret); + } + if (ret) { - drop: + wg_dbg("Failed to queue handshake packet, dropping\n"); + drop: net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", wg->dev->name, skb); goto err; } + atomic_inc(&wg->handshake_queue_len); + wg_dbg("Handshake queue length incremented\n"); + cpu = wg_cpumask_next_online(&wg->handshake_queue.last_cpu); + wg_dbg("Selected CPU for work queue: %d\n", cpu); + /* Queues up a call to packet_process_queued_handshake_packets(skb): */ queue_work_on(cpu, wg->handshake_receive_wq, &per_cpu_ptr(wg->handshake_queue.worker, cpu)->work); break; } case cpu_to_le32(MESSAGE_DATA): + wg_dbg("Received data packet\n"); PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); + wg_dbg("DS field set to: %u\n", PACKET_CB(skb)->ds); wg_packet_consume_data(wg, skb); break; default: WARN(1, "Non-exhaustive parsing of packet header lead to unknown packet type!\n"); + wg_dbg("Unknown packet type: %u, dropping\n", skb_type); goto err; } + + wg_dbg("Exiting wg_packet_receive normally\n"); return; err: dev_kfree_skb(skb); + wg_dbg("Exiting wg_packet_receive with error\n"); } diff --git a/kernel/selftest/allowedips.c b/kernel/selftest/allowedips.c index 3d1f64ff2e1225aea957f24c08b05e859956dc6f..6e341fc36dac33e6d5b5d7e884efb1133feaada2 100644 --- a/kernel/selftest/allowedips.c +++ b/kernel/selftest/allowedips.c @@ -461,6 +461,10 @@ static __init struct wg_peer *init_peer(void) wg_allowedips_insert_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \ cidr, mem, &mutex) +#define remove(version, mem, ipa, ipb, ipc, ipd, cidr) \ + wg_allowedips_remove_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \ + cidr, mem, &mutex) + #define maybe_fail() do { \ ++i; \ if (!_s) { \ @@ -586,6 +590,38 @@ bool __init wg_allowedips_selftest(void) test_negative(4, a, 192, 0, 0, 0); test_negative(4, a, 255, 0, 0, 0); + insert(4, a, 1, 0, 0, 0, 32); + insert(4, a, 192, 0, 0, 0, 24); + insert(6, a, 0x24446801, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); + insert(6, a, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + test(4, a, 1, 0, 0, 0); + test(4, a, 192, 0, 0, 1); + test(6, a, 0x24446801, 0x40e40800, 0xdeaebeef, 0xdefbeef); + test(6, a, 0x24446800, 0xf0e40800, 0xeeaebeef, 0x10101010); + remove(4, a, 192, 0, 0, 0, 32); + test(4, a, 192, 0, 0, 1); + test_boolean(!remove(4, NULL, 192, 0, 0, 0, 24)); + test_boolean(!remove(4, b, 192, 0, 0, 0, 24)); + test_boolean(remove(4, b, 192, 0, 0, 0, 33) == -EINVAL); + remove(4, a, 192, 0, 0, 0, 24); + test_negative(4, a, 192, 0, 0, 1); + remove(4, a, 1, 0, 0, 0, 32); + test_negative(4, a, 1, 0, 0, 0); + remove(6, a, 0x24446801, 0x40e40800, 0xdeaebeef, 0xdefbeef, 96); + test(6, a, 0x24446801, 0x40e40800, 0xdeaebeef, 0xdefbeef); + test_boolean(!remove(6, NULL, 0x24446801, 0x40e40800, + 0xdeaebeef, 0xdefbeef, 128)); + test_boolean(!remove(6, b, 0x24446801, 0x40e40800, + 0xdeaebeef, 0xdefbeef, 128)); + test_boolean(remove(6, a, 0x24446801, 0x40e40800, + 0xdeaebeef, 0xdefbeef, 129) == -EINVAL); + remove(6, a, 0x24446801, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); + test_negative(6, a, 0x24446801, 0x40e40800, 0xdeaebeef, 0xdefbeef); + remove(6, b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + test(6, a, 0x24446800, 0xf0e40800, 0xeeaebeef, 0x10101010); + remove(6, a, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + test_negative(6, a, 0x24446800, 0xf0e40800, 0xeeaebeef, 0x10101010); + wg_allowedips_free(&t, &mutex); wg_allowedips_init(&t); insert(4, a, 192, 168, 0, 0, 16); diff --git a/kernel/selftest/cookie.c b/kernel/selftest/cookie.c new file mode 100644 index 0000000000000000000000000000000000000000..8e792334c0c04c3cb7351fe2d518089ba2e57a0d --- /dev/null +++ b/kernel/selftest/cookie.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifdef DEBUG +bool __init wg_cookie_policy_selftest(void) +{ + static const struct { + bool under_load; + enum cookie_mac_state state; + enum cookie_validation_action expected; + } cases[] = { + { false, INVALID_MAC, WG_COOKIE_DROP }, + { false, VALID_MAC_BUT_NO_COOKIE, WG_COOKIE_ACCEPT }, + { false, VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, WG_COOKIE_DROP }, + { false, VALID_MAC_WITH_COOKIE, WG_COOKIE_DROP }, + { true, INVALID_MAC, WG_COOKIE_DROP }, + { true, VALID_MAC_BUT_NO_COOKIE, WG_COOKIE_CHALLENGE }, + { true, VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, WG_COOKIE_DROP }, + { true, VALID_MAC_WITH_COOKIE, WG_COOKIE_ACCEPT } + }; + size_t i; + + for (i = 0; i < ARRAY_SIZE(cases); ++i) { + if (wg_cookie_validation_action(cases[i].under_load, + cases[i].state) != cases[i].expected) { + pr_err("cookie policy self-test %zu: FAIL\n", i + 1); + return false; + } + } + pr_info("cookie policy self-tests: pass\n"); + return true; +} +#endif diff --git a/kernel/send.c b/kernel/send.c index 0d48e0f4a1ba3e1f11825136a65de0867b204496..5b0cdf72548620f76d43b6c5b6bae8e362a1555c 100644 --- a/kernel/send.c +++ b/kernel/send.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ #include "queueing.h" @@ -8,53 +9,105 @@ #include "device.h" #include "peer.h" #include "socket.h" +#include "wg_tcp.h" #include "messages.h" #include "cookie.h" +#include "wg_tcp_debug.h" #include #include #include +#include #include #include #include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + static void wg_packet_send_handshake_initiation(struct wg_peer *peer) { struct message_handshake_initiation packet; - if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), - REKEY_TIMEOUT)) - return; /* This function is rate limited. */ + /* Enter function with peer information */ + wg_dbg("Entering wg_packet_send_handshake_initiation with peer=%px\n", peer); + + /* Check rate limiting */ + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), REKEY_TIMEOUT)) { + wg_dbg("wg_packet_send_handshake_initiation: Handshake rate limit not expired for peer=%px\n", peer); + wg_dbg("Exiting wg_packet_send_handshake_initiation\n"); + return; + } + /* Update last sent handshake time */ atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); - net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr); + /* Log handshake initiation attempt */ + wg_dbg("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + /* Create handshake initiation */ if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { + wg_dbg("wg_packet_send_handshake_initiation: Handshake initiation created successfully for peer=%px\n", peer); + + /* Print out the contents of the handshake initiation packet */ + wg_dbg("wg_packet_send_handshake_initiation: Handshake packet contents: %*ph\n", + (int)sizeof(packet), &packet); + + /* Add MAC to packet */ wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + wg_dbg("wg_packet_send_handshake_initiation: MAC added to handshake packet for peer=%px\n", peer); + + /* Timers and sending operations */ wg_timers_any_authenticated_packet_traversal(peer); wg_timers_any_authenticated_packet_sent(peer); - atomic64_set(&peer->last_sent_handshake, - ktime_get_coarse_boottime_ns()); - wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), - HANDSHAKE_DSCP); + + /* Update last sent handshake time again */ + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + + /* Send the handshake packet */ + wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), HANDSHAKE_DSCP); + wg_dbg("wg_packet_send_handshake_initiation: Handshake packet sent to peer=%px\n", peer); + + /* Mark handshake initiation complete */ wg_timers_handshake_initiated(peer); + } else { + /* Log failure to create handshake initiation */ + wg_dbg("wg_packet_send_handshake_initiation: Failed to create handshake initiation for peer=%px\n", peer); } + + /* Exit function */ + wg_dbg("Exiting wg_packet_send_handshake_initiation\n"); } void wg_packet_handshake_send_worker(struct work_struct *work) { + wg_dbg("Entering wg_packet_handshake_send_worker with work=%px\n", work); struct wg_peer *peer = container_of(work, struct wg_peer, transmit_handshake_work); wg_packet_send_handshake_initiation(peer); wg_peer_put(peer); + wg_dbg("Exiting wg_packet_handshake_send_worker\n"); } void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, bool is_retry) { + wg_dbg("Entering wg_packet_send_queued_handshake_initiation with peer=%px, is_retry=%d\n", peer, is_retry); if (!is_retry) peer->timer_handshake_attempts = 0; @@ -80,10 +133,12 @@ void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, wg_peer_put(peer); out: rcu_read_unlock_bh(); + wg_dbg("Exiting wg_packet_send_queued_handshake_initiation\n"); } void wg_packet_send_handshake_response(struct wg_peer *peer) { + wg_dbg("Entering wg_packet_send_handshake_response with peer=%px\n", peer); struct message_handshake_response packet; atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); @@ -93,6 +148,14 @@ void wg_packet_send_handshake_response(struct wg_peer *peer) if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + + wg_dbg("MAC added to handshake response packet\n"); + wg_dbg("Handshake Response Packet: %*ph\n", + (int)sizeof(packet), &packet); + wg_dbg("Peer Cookie Parameters: peer=%px, handshake=%px, index=%u\n", + peer, &peer->handshake, packet.sender_index); + + if (wg_noise_handshake_begin_session(&peer->handshake, &peer->keypairs)) { wg_timers_session_derived(peer); @@ -105,24 +168,41 @@ void wg_packet_send_handshake_response(struct wg_peer *peer) HANDSHAKE_DSCP); } } + wg_dbg("Exiting wg_packet_send_handshake_response\n"); } void wg_packet_send_handshake_cookie(struct wg_device *wg, struct sk_buff *initiating_skb, __le32 sender_index) { + wg_dbg("Entering wg_packet_send_handshake_cookie with wg=%px, initiating_skb=%px, sender_index=%u\n", wg, initiating_skb, sender_index); struct message_handshake_cookie packet; + wg_dbg("Creating handshake cookie\n"); + wg_dbg("initiating_skb len=%u\n", initiating_skb->len); + wg_dbg("Initiating SKB Data: %*ph\n", + (int)initiating_skb->len, initiating_skb->data); + + wg_dbg("Cookie Checker: %px\n", &wg->cookie_checker); + + net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", wg->dev->name, initiating_skb); wg_cookie_message_create(&packet, initiating_skb, sender_index, &wg->cookie_checker); + + wg_dbg("Handshake Cookie Packet: %*ph\n", + (int)sizeof(packet), &packet); + wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, sizeof(packet)); + + wg_dbg("Exiting wg_packet_send_handshake_cookie\n"); } static void keep_key_fresh(struct wg_peer *peer) { + wg_dbg("Entering keep_key_fresh with peer=%px\n", peer); struct noise_keypair *keypair; bool send; @@ -136,14 +216,18 @@ static void keep_key_fresh(struct wg_peer *peer) if (unlikely(send)) wg_packet_send_queued_handshake_initiation(peer, false); + wg_dbg("Exiting keep_key_fresh\n"); } static unsigned int calculate_skb_padding(struct sk_buff *skb) { + wg_dbg("Entering calculate_skb_padding with skb=%px\n", skb); unsigned int padded_size, last_unit = skb->len; - if (unlikely(!PACKET_CB(skb)->mtu)) + if (unlikely(!PACKET_CB(skb)->mtu)) { + wg_dbg("Exiting calculate_skb_padding\n"); return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit; + } /* We do this modulo business with the MTU, just in case the networking * layer gives us a packet that's bigger than the MTU. In that case, we @@ -156,77 +240,130 @@ static unsigned int calculate_skb_padding(struct sk_buff *skb) padded_size = min(PACKET_CB(skb)->mtu, ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE)); + wg_dbg("Exiting calculate_skb_padding\n"); return padded_size - last_unit; } static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) { - unsigned int padding_len, plaintext_len, trailer_len; - struct scatterlist sg[MAX_SKB_FRAGS + 8]; - struct message_data *header; - struct sk_buff *trailer; - int num_frags; - - /* Force hash calculation before encryption so that flow analysis is - * consistent over the inner packet. - */ - skb_get_hash(skb); + unsigned int padding_len, plaintext_len, trailer_len; + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct message_data *header; + struct sk_buff *trailer; + int num_frags; + + wg_dbg("Entering encrypt_packet with skb=%px, keypair=%px\n", skb, keypair); + wg_dbg("skb->len = %u, skb->data_len = %u, skb->network_header = %px\n", skb->len, skb->data_len, skb_network_header(skb)); + wg_dbg("keypair->remote_index = %u\n", keypair->remote_index); + wg_dbg("skb->data (before encryption): %*ph\n", skb->len, skb->data); +#ifdef WG_TCP_VERBOSE + decode_and_print_packet(skb, "[encrypt]"); +#endif + + /* Force hash calculation before encryption */ + skb_get_hash(skb); + wg_dbg("Hash calculated for skb.\n"); + + /* Calculate lengths */ + padding_len = calculate_skb_padding(skb); + trailer_len = padding_len + noise_encrypted_len(0); + plaintext_len = skb->len + padding_len; + wg_dbg("Calculated lengths: padding_len=%u, trailer_len=%u, plaintext_len=%u\n", padding_len, trailer_len, plaintext_len); + + /* Expand head section */ + if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) { + wg_dbg("Failed skb_cow_head, skb->len=%u, skb->head=%px\n", skb->len, skb->head); + wg_dbg("Exiting encrypt_packet with false\n"); + return false; + } + wg_dbg("Expanded head section, skb->len=%u, skb->head=%px\n", skb->len, skb->head); + + /* Finalize checksum calculation */ + if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb))) { + wg_dbg("Failed skb_checksum_help, skb->len=%u\n", skb->len); + wg_dbg("Exiting encrypt_packet with false\n"); + return false; + } + wg_dbg("Checksum finalized, skb->len=%u\n", skb->len); + + /* Expand data only after checksum completion; skb_checksum_help() may + * alter the packet layout and invalidate an earlier trailer pointer. + */ + num_frags = skb_cow_data(skb, trailer_len, &trailer); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) { + wg_dbg("Failed skb_cow_data: num_frags=%d, skb->len=%u\n", num_frags, skb->len); + wg_dbg("Exiting encrypt_packet with false\n"); + return false; + } + + memset(skb_tail_pointer(trailer), 0, padding_len); + wg_dbg("Padding set to zeros: padding_len=%u, skb->len=%u\n", padding_len, skb->len); + + /* Add padding and header */ + skb_set_inner_network_header(skb, 0); + header = (struct message_data *)skb_push(skb, sizeof(*header)); + header->header.type = cpu_to_le32(MESSAGE_DATA); + header->key_idx = keypair->remote_index; + header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); + wg_dbg("Nonce for encryption: %llu\n", PACKET_CB(skb)->nonce); +#define NOISE_KEY_LEN 32 + wg_dbg("Encryption key: %*ph\n", NOISE_KEY_LEN, keypair->sending.key); + pskb_put(skb, trailer, trailer_len); + wg_dbg("Header and padding added: type=%u, key_idx=%u, counter=%llu\n", + MESSAGE_DATA, keypair->remote_index, PACKET_CB(skb)->nonce); + wg_dbg("Network header set: skb_network_header=%px, skb->len=%u\n", skb_network_header(skb), skb->len); + + /* Encrypt the scattergather segments */ + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, sizeof(struct message_data), noise_encrypted_len(plaintext_len)) <= 0) { + wg_dbg("Failed skb_to_sgvec, skb->len=%u\n", skb->len); + wg_dbg("Exiting encrypt_packet with false\n"); + return false; + } + + wg_dbg("Scattergather segments prepared, starting encryption\n"); + + bool success = chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, + PACKET_CB(skb)->nonce, + keypair->sending.key); + wg_dbg("skb->data (after encryption): %*ph\n", skb->len, skb->data); + wg_dbg("Exiting encrypt_packet with %s, skb->len=%u\n", success ? "true" : "false", skb->len); + return success; +} - /* Calculate lengths. */ - padding_len = calculate_skb_padding(skb); - trailer_len = padding_len + noise_encrypted_len(0); - plaintext_len = skb->len + padding_len; +/* Helper function to extract IPv4 fragmentation info */ +static inline bool wg_ipv4_get_fraginfo(const struct sk_buff *skb, + __be16 *id, __be16 *frag_off) +{ + const struct iphdr *iph; - /* Expand data section to have room for padding and auth tag. */ - num_frags = skb_cow_data(skb, trailer_len, &trailer); - if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + if (skb->protocol != htons(ETH_P_IP)) return false; - /* Set the padding to zeros, and make sure it and the auth tag are part - * of the skb. - */ - memset(skb_tail_pointer(trailer), 0, padding_len); - - /* Expand head section to have room for our header and the network - * stack's headers. - */ - if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) + if (!pskb_may_pull((struct sk_buff *)skb, sizeof(struct iphdr))) return false; - /* Finalize checksum calculation for the inner packet, if required. */ - if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && - skb_checksum_help(skb))) - return false; + iph = ip_hdr(skb); + if (!(iph->frag_off & htons(IP_MF | IP_OFFSET))) + return false; /* not fragmented */ - /* Only after checksumming can we safely add on the padding at the end - * and the header. - */ - skb_set_inner_network_header(skb, 0); - header = (struct message_data *)skb_push(skb, sizeof(*header)); - header->header.type = cpu_to_le32(MESSAGE_DATA); - header->key_idx = keypair->remote_index; - header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); - pskb_put(skb, trailer, trailer_len); - - /* Now we can encrypt the scattergather segments */ - sg_init_table(sg, num_frags); - if (skb_to_sgvec(skb, sg, sizeof(struct message_data), - noise_encrypted_len(plaintext_len)) <= 0) - return false; - return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, - PACKET_CB(skb)->nonce, - keypair->sending.key); + *id = iph->id; + *frag_off = iph->frag_off; + return true; } void wg_packet_send_keepalive(struct wg_peer *peer) { + wg_dbg("Entering wg_packet_send_keepalive with peer=%px\n", peer); struct sk_buff *skb; - if (skb_queue_empty(&peer->staged_packet_queue)) { + if (skb_queue_empty_lockless(&peer->staged_packet_queue)) { skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, GFP_ATOMIC); - if (unlikely(!skb)) + if (unlikely(!skb)) { + wg_dbg("Exiting wg_packet_send_keepalive\n"); return; + } skb_reserve(skb, DATA_PACKET_HEAD_ROOM); skb->dev = peer->device->dev; PACKET_CB(skb)->mtu = skb->dev->mtu; @@ -237,10 +374,12 @@ void wg_packet_send_keepalive(struct wg_peer *peer) } wg_packet_send_staged_packets(peer); + wg_dbg("Exiting wg_packet_send_keepalive\n"); } static void wg_packet_create_data_done(struct wg_peer *peer, struct sk_buff *first) { + wg_dbg("Entering wg_packet_create_data_done with peer=%px, first=%px\n", peer, first); struct sk_buff *skb, *next; bool is_keepalive, data_sent = false; @@ -257,10 +396,12 @@ static void wg_packet_create_data_done(struct wg_peer *peer, struct sk_buff *fir wg_timers_data_sent(peer); keep_key_fresh(peer); + wg_dbg("Exiting wg_packet_create_data_done\n"); } void wg_packet_tx_worker(struct work_struct *work) { + wg_dbg("Entering wg_packet_tx_worker with work=%px\n", work); struct wg_peer *peer = container_of(work, struct wg_peer, transmit_packet_work); struct noise_keypair *keypair; enum packet_state state; @@ -282,10 +423,12 @@ void wg_packet_tx_worker(struct work_struct *work) if (need_resched()) cond_resched(); } + wg_dbg("Exiting wg_packet_tx_worker\n"); } void wg_packet_encrypt_worker(struct work_struct *work) { + wg_dbg("Entering wg_packet_encrypt_worker with work=%px\n", work); struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr; struct sk_buff *first, *skb, *next; @@ -306,10 +449,12 @@ void wg_packet_encrypt_worker(struct work_struct *work) if (need_resched()) cond_resched(); } + wg_dbg("Exiting wg_packet_encrypt_worker\n"); } static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first) { + wg_dbg("Entering wg_packet_create_data with peer=%px, first=%px\n", peer, first); struct wg_device *wg = peer->device; int ret = -EINVAL; @@ -317,41 +462,50 @@ static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first) if (unlikely(READ_ONCE(peer->is_dead))) goto err; + wg_dbg("wg_packet_create_data sending: %*ph\n", first->len, first->data); + ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, &peer->tx_queue, first, wg->packet_crypt_wq); if (unlikely(ret == -EPIPE)) wg_queue_enqueue_per_peer_tx(first, PACKET_STATE_DEAD); + err: rcu_read_unlock_bh(); - if (likely(!ret || ret == -EPIPE)) + if (likely(!ret || ret == -EPIPE)) { + wg_dbg("Exiting wg_packet_create_data\n"); return; + } wg_noise_keypair_put(PACKET_CB(first)->keypair, false); wg_peer_put(peer); kfree_skb_list(first); + wg_dbg("Exiting wg_packet_create_data with error.\n"); } void wg_packet_purge_staged_packets(struct wg_peer *peer) { + wg_dbg("Entering wg_packet_purge_staged_packets with peer=%px\n", peer); spin_lock_bh(&peer->staged_packet_queue.lock); DEV_STATS_ADD(peer->device->dev, tx_dropped, peer->staged_packet_queue.qlen); __skb_queue_purge(&peer->staged_packet_queue); spin_unlock_bh(&peer->staged_packet_queue.lock); + wg_dbg("Exiting wg_packet_purge_staged_packets\n"); } void wg_packet_send_staged_packets(struct wg_peer *peer) { + wg_dbg("Entering wg_packet_send_staged_packets with peer=%px\n", peer); struct noise_keypair *keypair; struct sk_buff_head packets; struct sk_buff *skb; - - /* Steal the current queue into our local one. */ __skb_queue_head_init(&packets); spin_lock_bh(&peer->staged_packet_queue.lock); skb_queue_splice_init(&peer->staged_packet_queue, &packets); spin_unlock_bh(&peer->staged_packet_queue.lock); - if (unlikely(skb_queue_empty(&packets))) + if (unlikely(skb_queue_empty(&packets))) { + wg_dbg("Exiting wg_packet_send_staged_packets\n"); return; + } /* First we make sure we have a valid reference to a valid key. */ rcu_read_lock_bh(); @@ -380,12 +534,38 @@ void wg_packet_send_staged_packets(struct wg_peer *peer) atomic64_inc_return(&keypair->sending_counter) - 1; if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) goto out_invalid; + + /* XXX - Jeff: + * This codepath is only executed for for keepalives, + * when an interface comes up, or when set_peer is called. + * This is likely unnecessary and we'll revisit it + * for removal + */ + /* Extract fragmentation info if this is IPv4 and fragmented */ + if (peer->device->transport == WG_TRANSPORT_TCP && + skb->protocol == htons(ETH_P_IP)) { + __be16 id = 0, frag_off = 0; + if (wg_ipv4_get_fraginfo(skb, &id, &frag_off)) { + PACKET_CB(skb)->frag_id = id; + PACKET_CB(skb)->frag_off = frag_off; + wg_dbg("Fragmentation detected: id=%u, frag_off=0x%x\n", + ntohs(id), ntohs(frag_off)); + } else { + PACKET_CB(skb)->frag_id = 0; + PACKET_CB(skb)->frag_off = 0; + } + } else { + /* Non-IPv4 packets don't have fragmentation info */ + PACKET_CB(skb)->frag_id = 0; + PACKET_CB(skb)->frag_off = 0; + } } packets.prev->next = NULL; wg_peer_get(keypair->entry.peer); PACKET_CB(packets.next)->keypair = keypair; wg_packet_create_data(peer, packets.next); + wg_dbg("Exiting wg_packet_send_staged_packets\n"); return; out_invalid: @@ -411,4 +591,5 @@ out_nokey: * means we should initiate a new handshake. */ wg_packet_send_queued_handshake_initiation(peer, false); + wg_dbg("Exiting wg_packet_send_staged_packets\n"); } diff --git a/kernel/socket.c b/kernel/socket.c index 0414d7a6ce74141cd2ca365bfd1da727691e27ec..ccf98aa7e1f9c52fd002a1c4074b053a89ffeec3 100644 --- a/kernel/socket.c +++ b/kernel/socket.c @@ -1,25 +1,58 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ + #include "device.h" #include "peer.h" #include "socket.h" +#include "wg_tcp.h" #include "queueing.h" #include "messages.h" +#include /* For ntohl */ #include -#include +#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include "wg_tcp_debug.h" static int send4(struct wg_device *wg, struct sk_buff *skb, struct endpoint *endpoint, u8 ds, struct dst_cache *cache) { + wg_dbg("Entering function send4\n"); + + struct flowi4 fl = { .saddr = endpoint->src4.s_addr, .daddr = endpoint->addr4.sin_addr.s_addr, @@ -71,11 +104,17 @@ static int send4(struct wg_device *wg, struct sk_buff *skb, ip_rt_put(rt); rt = ip_route_output_flow(sock_net(sock), &fl, sock); } - if (IS_ERR(rt)) { + if (unlikely(IS_ERR(rt))) { ret = PTR_ERR(rt); net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", wg->dev->name, &endpoint->addr, ret); goto err; + } else if (unlikely(rt->dst.dev == skb->dev)) { + ip_rt_put(rt); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; } if (cache) dst_cache_set_ip4(cache, &rt->dst, fl.saddr); @@ -91,13 +130,17 @@ err: kfree_skb(skb); out: rcu_read_unlock_bh(); + wg_dbg("Exiting function send4\n"); return ret; } static int send6(struct wg_device *wg, struct sk_buff *skb, struct endpoint *endpoint, u8 ds, struct dst_cache *cache) { + wg_dbg("Entering function send6\n"); #if IS_ENABLED(CONFIG_IPV6) + + struct flowi6 fl = { .saddr = endpoint->src6, .daddr = endpoint->addr6.sin6_addr, @@ -138,11 +181,17 @@ static int send6(struct wg_device *wg, struct sk_buff *skb, } dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, NULL); - if (IS_ERR(dst)) { + if (unlikely(IS_ERR(dst))) { ret = PTR_ERR(dst); net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", wg->dev->name, &endpoint->addr, ret); goto err; + } else if (unlikely(dst->dev == skb->dev)) { + dst_release(dst); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; } if (cache) dst_cache_set_ip6(cache, dst, &fl.saddr); @@ -158,52 +207,68 @@ err: kfree_skb(skb); out: rcu_read_unlock_bh(); + wg_dbg("Exiting function send6\n"); return ret; #else kfree_skb(skb); + wg_dbg("Exiting function send6\n"); return -EAFNOSUPPORT; #endif } -int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) +int wg_socket_send_skb_to_endpoint(struct wg_device *wg, + struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, + struct dst_cache *cache) { - size_t skb_len = skb->len; - int ret = -EAFNOSUPPORT; - - read_lock_bh(&peer->endpoint_lock); - if (peer->endpoint.addr.sa_family == AF_INET) - ret = send4(peer->device, skb, &peer->endpoint, ds, - &peer->endpoint_cache); - else if (peer->endpoint.addr.sa_family == AF_INET6) - ret = send6(peer->device, skb, &peer->endpoint, ds, - &peer->endpoint_cache); - else - dev_kfree_skb(skb); - if (likely(!ret)) - peer->tx_bytes += skb_len; - read_unlock_bh(&peer->endpoint_lock); + if (endpoint->addr.sa_family == AF_INET) + return send4(wg, skb, endpoint, ds, cache); + if (endpoint->addr.sa_family == AF_INET6) + return send6(wg, skb, endpoint, ds, cache); - return ret; + dev_kfree_skb(skb); + return -EAFNOSUPPORT; } int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer, size_t len, u8 ds) { - struct sk_buff *skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + int ret; + struct sk_buff *skb; - if (unlikely(!skb)) - return -ENOMEM; + wg_dbg("Entering function wg_socket_send_buffer_to_peer peer=%px\n", peer); + + /* BUG FIX: null check peer BEFORE dereferencing it */ + if (unlikely(!peer) || unlikely(IS_ERR(peer))) { + ret = -EINVAL; + goto out; + } + + log_wireguard_endpoint(&peer->endpoint); + skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + + wg_dbg("Sending buffer to peer - Length: %zu, Data: %*ph\n", + len, (int)len, buffer); + if (unlikely(!skb)){ + ret = -ENOMEM; + goto out; + } skb_reserve(skb, SKB_HEADER_LEN); skb_set_inner_network_header(skb, 0); skb_put_data(skb, buffer, len); - return wg_socket_send_skb_to_peer(peer, skb, ds); + ret = wg_socket_send_skb_to_peer(peer, skb, ds); + +out: + wg_dbg("Exiting function wg_socket_send_buffer_to_peer\n"); + return ret; } int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, struct sk_buff *in_skb, void *buffer, size_t len) { + wg_dbg("Entering function wg_socket_send_buffer_as_reply_to_skb\n"); int ret = 0; struct sk_buff *skb; struct endpoint endpoint; @@ -221,20 +286,19 @@ int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, skb_set_inner_network_header(skb, 0); skb_put_data(skb, buffer, len); - if (endpoint.addr.sa_family == AF_INET) - ret = send4(wg, skb, &endpoint, 0, NULL); - else if (endpoint.addr.sa_family == AF_INET6) - ret = send6(wg, skb, &endpoint, 0, NULL); - /* No other possibilities if the endpoint is valid, which it is, - * as we checked above. - */ + ret = wg_socket_send_skb_to_endpoint(wg, skb, &endpoint, 0, NULL); + wg_dbg("Exiting function wg_socket_send_buffer_as_reply_to_skb\n"); return ret; } -int wg_socket_endpoint_from_skb(struct endpoint *endpoint, - const struct sk_buff *skb) + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, const struct sk_buff *skb) { + wg_dbg("Entering function wg_socket_endpoint_from_skb\n"); + + wg_dbg("skb data: %*ph\n", min_t(int, skb->len, 128), skb->data); + memset(endpoint, 0, sizeof(*endpoint)); if (skb->protocol == htons(ETH_P_IP)) { endpoint->addr4.sin_family = AF_INET; @@ -242,21 +306,29 @@ int wg_socket_endpoint_from_skb(struct endpoint *endpoint, endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr; endpoint->src4.s_addr = ip_hdr(skb)->daddr; endpoint->src_if4 = skb->skb_iif; + wg_dbg("wg_socket_endpoint_from_skb: Extracted IPv4 address %pI4:%d\n", + &endpoint->addr4.sin_addr, ntohs(endpoint->addr4.sin_port)); } else if (IS_ENABLED(CONFIG_IPV6) && skb->protocol == htons(ETH_P_IPV6)) { endpoint->addr6.sin6_family = AF_INET6; endpoint->addr6.sin6_port = udp_hdr(skb)->source; endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr; - endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id( - &ipv6_hdr(skb)->saddr, skb->skb_iif); + endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif); endpoint->src6 = ipv6_hdr(skb)->daddr; + wg_dbg("wg_socket_endpoint_from_skb: Extracted IPv6 address %pI6c:%d\n", + &endpoint->addr6.sin6_addr, ntohs(endpoint->addr6.sin6_port)); } else { return -EINVAL; } + + + wg_dbg("Exiting function wg_socket_endpoint_from_skb\n"); return 0; } -static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) +bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) { + wg_dbg("Entering function endpoint_eq\n"); + wg_dbg("Exiting function endpoint_eq\n"); return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET && a->addr4.sin_port == b->addr4.sin_port && a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr && @@ -269,57 +341,32 @@ static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) ipv6_addr_equal(&a->src6, &b->src6)) || unlikely(!a->addr.sa_family && !b->addr.sa_family); } - -void wg_socket_set_peer_endpoint(struct wg_peer *peer, - const struct endpoint *endpoint) +static void sock_free(struct sock *sock) { - /* First we check unlocked, in order to optimize, since it's pretty rare - * that an endpoint will change. If we happen to be mid-write, and two - * CPUs wind up writing the same thing or something slightly different, - * it doesn't really matter much either. - */ - if (endpoint_eq(endpoint, &peer->endpoint)) + wg_dbg("Entering function sock_free\n"); + if (unlikely(!sock)) return; - write_lock_bh(&peer->endpoint_lock); - if (endpoint->addr.sa_family == AF_INET) { - peer->endpoint.addr4 = endpoint->addr4; - peer->endpoint.src4 = endpoint->src4; - peer->endpoint.src_if4 = endpoint->src_if4; - } else if (IS_ENABLED(CONFIG_IPV6) && endpoint->addr.sa_family == AF_INET6) { - peer->endpoint.addr6 = endpoint->addr6; - peer->endpoint.src6 = endpoint->src6; - } else { - goto out; - } - dst_cache_reset(&peer->endpoint_cache); -out: - write_unlock_bh(&peer->endpoint_lock); -} - -void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, - const struct sk_buff *skb) -{ - struct endpoint endpoint; - - if (!wg_socket_endpoint_from_skb(&endpoint, skb)) - wg_socket_set_peer_endpoint(peer, &endpoint); + sk_clear_memalloc(sock); + udp_tunnel_sock_release(sock->sk_socket); + wg_dbg("Exiting function sock_free\n"); } -void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) +static void set_sock_opts(struct socket *sock) { - write_lock_bh(&peer->endpoint_lock); - memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); - dst_cache_reset_now(&peer->endpoint_cache); - write_unlock_bh(&peer->endpoint_lock); + wg_dbg("Entering function set_sock_opts\n"); + sock->sk->sk_allocation = GFP_ATOMIC; + sock->sk->sk_sndbuf = INT_MAX; + sk_set_memalloc(sock->sk); + wg_dbg("Exiting function set_sock_opts\n"); } -static int wg_receive(struct sock *sk, struct sk_buff *skb) +static int wg_udp_receive(struct sock *sk, struct sk_buff *skb) { struct wg_device *wg; if (unlikely(!sk)) goto err; - wg = sk->sk_user_data; + wg = READ_ONCE(sk->sk_user_data); if (unlikely(!wg)) goto err; skb_mark_not_on_list(skb); @@ -331,29 +378,15 @@ err: return 0; } -static void sock_free(struct sock *sock) -{ - if (unlikely(!sock)) - return; - sk_clear_memalloc(sock); - udp_tunnel_sock_release(sock->sk_socket); -} - -static void set_sock_opts(struct socket *sock) -{ - sock->sk->sk_allocation = GFP_ATOMIC; - sock->sk->sk_sndbuf = INT_MAX; - sk_set_memalloc(sock->sk); -} - int wg_socket_init(struct wg_device *wg, u16 port) { + wg_dbg("Entering function wg_socket_init\n"); struct net *net; int ret; struct udp_tunnel_sock_cfg cfg = { .sk_user_data = wg, .encap_type = 1, - .encap_rcv = wg_receive + .encap_rcv = wg_udp_receive }; struct socket *new4 = NULL, *new6 = NULL; struct udp_port_cfg port4 = { @@ -390,6 +423,7 @@ retry: goto out; } set_sock_opts(new4); + setup_udp_tunnel_sock(net, new4, &cfg); #if IS_ENABLED(CONFIG_IPV6) @@ -405,6 +439,8 @@ retry: goto out; } set_sock_opts(new6); + + /* Setup the IPv6 UDP tunnel socket with the same socket data */ setup_udp_tunnel_sock(net, new6, &cfg); } #endif @@ -413,12 +449,15 @@ retry: ret = 0; out: put_net(net); + wg_dbg("Exiting function wg_socket_init\n"); return ret; } + void wg_socket_reinit(struct wg_device *wg, struct sock *new4, struct sock *new6) { + wg_dbg("Entering function wg_socket_reinit\n"); struct sock *old4, *old6; mutex_lock(&wg->socket_update_lock); @@ -431,7 +470,9 @@ void wg_socket_reinit(struct wg_device *wg, struct sock *new4, if (new4) wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); mutex_unlock(&wg->socket_update_lock); + synchronize_rcu(); synchronize_net(); sock_free(old4); sock_free(old6); + wg_dbg("Exiting function wg_socket_reinit\n"); } diff --git a/kernel/socket.h b/kernel/socket.h index bab5848efbcdfe59056115c132caa0f618148288..166674686fe904518069ca33939c7717ea31bc3d 100644 --- a/kernel/socket.h +++ b/kernel/socket.h @@ -1,11 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024 Jeff Nathan and Dragos Ruiu. All Rights Reserved. */ #ifndef _WG_SOCKET_H #define _WG_SOCKET_H +#include #include #include #include @@ -16,19 +18,53 @@ void wg_socket_reinit(struct wg_device *wg, struct sock *new4, struct sock *new6); int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *data, size_t len, u8 ds); -int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, - u8 ds); int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, struct sk_buff *in_skb, void *out_buffer, size_t len); int wg_socket_endpoint_from_skb(struct endpoint *endpoint, const struct sk_buff *skb); -void wg_socket_set_peer_endpoint(struct wg_peer *peer, - const struct endpoint *endpoint); -void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, - const struct sk_buff *skb); -void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); +int wg_socket_send_skb_to_endpoint(struct wg_device *wg, + struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, + struct dst_cache *cache); + +struct wg_tcp_encap_header { + __be32 length; + __u8 type; + __u8 flags; + __be16 checksum; +}; + +struct wg_tcp_frag_header { + __be16 id; + __be16 frag_off; +}; + +struct wg_tcp_socket_list_entry { + struct socket *tcp_socket; /* Socket associated with the connection */ + struct sockaddr_storage src_addr; /* Source address for the connection */ + struct wg_peer *temp_peer; /* temporary peer for dataready */ + struct list_head tcp_connection_ll; /* List pointer for the linked list */ + ktime_t created_at; /* Absolute pre-authentication deadline base */ + ktime_t timestamp; /* Most recent pre-authentication activity */ + u64 connection_id; /* Stable carrier identity across async auth */ + bool authenticated; /* Exact stream carried valid Noise traffic */ + bool admission_counted; /* Owns one pre-authentication reservation */ + bool initializing; /* Listener still owns callback handoff */ +}; + +#define WG_TCP_ENCAP_HDR_LEN sizeof(struct wg_tcp_encap_header) +#define WG_TCP_FRAG_HDR_LEN sizeof(struct wg_tcp_frag_header) +#define WG_MAX_PACKET_SIZE 65535 + WG_TCP_ENCAP_HDR_LEN +#define WG_TCP_SKB_READ_ALLOC_SIZE 8192 * 3 +/* A nominal 128 bytes to account for various + * stacked headers in any given Ethernet frame + */ +#define WG_TCP_RESERVED_HEADER_SIZE 128 +#define WG_TCP_RECORD_DATA 0 +/* Flags */ +#define WG_TCP_FRAG_FLAG 0x1 #if defined(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG) #define net_dbg_skb_ratelimited(fmt, dev, skb, ...) do { \ @@ -41,4 +77,9 @@ void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); #define net_dbg_skb_ratelimited(fmt, skb, ...) #endif +void wg_tcp_connection_retry_timer(struct timer_list *); + +bool endpoint_eq(const struct endpoint *a, const struct endpoint *b); +void wg_print_wireguard_skb(const struct sk_buff *skb); + #endif /* _WG_SOCKET_H */ diff --git a/kernel/timers.c b/kernel/timers.c index 968bdb4df0b300d12abf6fe4e1858225b3caf044..5daf66f957c4e513451a7f55616cc78e90837c9e 100644 --- a/kernel/timers.c +++ b/kernel/timers.c @@ -8,6 +8,7 @@ #include "peer.h" #include "queueing.h" #include "socket.h" +#include "wg_tcp.h" /* * - Timer for retransmitting the handshake if we don't hear back after diff --git a/kernel/wg_tcp.c b/kernel/wg_tcp.c new file mode 100644 index 0000000000000000000000000000000000000000..4e6ffb768589a3854e5264133d72ac4fafbcf4c5 --- /dev/null +++ b/kernel/wg_tcp.c @@ -0,0 +1,4979 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. + */ +#include + +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "wg_tcp.h" +#include "queueing.h" +#include "messages.h" + +#include /* For ntohl */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "wg_tcp_debug.h" + + +#if defined(DEBUG) && defined(WG_TCP_FAULT_INJECTION) +#define WG_TCP_TEST_MAX_GARBAGE_PREFIX 64 +#define WG_TCP_TEST_MAX_WRITE_DELAY_MS 1000 + +static unsigned int wg_tcp_test_max_send_bytes; +static unsigned int wg_tcp_test_garbage_prefix_bytes; +static unsigned int wg_tcp_test_queue_limit; +static unsigned int wg_tcp_test_write_delay_ms; +static unsigned int wg_tcp_test_fail_send_netns; +static unsigned int wg_tcp_test_fail_send_ifindex; +static unsigned int wg_tcp_test_fail_send_local_ipv4; +static unsigned int wg_tcp_test_fail_send_source_port; +static unsigned int wg_tcp_test_fail_send_remote_ipv4; +static unsigned int wg_tcp_test_fail_send_remote_port; +static unsigned int wg_tcp_test_fail_next_send; +static atomic64_t wg_tcp_test_short_writes = ATOMIC64_INIT(0); +static atomic64_t wg_tcp_test_injected_prefixes = ATOMIC64_INIT(0); +static atomic64_t wg_tcp_test_resyncs = ATOMIC64_INIT(0); +static atomic64_t wg_tcp_test_queue_drops = ATOMIC64_INIT(0); +static atomic64_t wg_tcp_test_fatal_send_errors = ATOMIC64_INIT(0); + +module_param_named(tcp_test_max_send_bytes, wg_tcp_test_max_send_bytes, + uint, 0600); +MODULE_PARM_DESC(tcp_test_max_send_bytes, + "DEBUG only: cap each TCP sendmsg request to force short writes"); +module_param_named(tcp_test_garbage_prefix_bytes, + wg_tcp_test_garbage_prefix_bytes, uint, 0600); +MODULE_PARM_DESC(tcp_test_garbage_prefix_bytes, + "DEBUG only: prepend bounded garbage to each TCP record"); +module_param_named(tcp_test_queue_limit, wg_tcp_test_queue_limit, uint, 0600); +MODULE_PARM_DESC(tcp_test_queue_limit, + "DEBUG only: lower the per-peer TCP frame queue limit"); +module_param_named(tcp_test_write_delay_ms, wg_tcp_test_write_delay_ms, + uint, 0600); +MODULE_PARM_DESC(tcp_test_write_delay_ms, + "DEBUG only: delay the next serial TCP writer to force queue pressure"); +module_param_named(tcp_test_fail_send_netns, wg_tcp_test_fail_send_netns, + uint, 0600); +MODULE_PARM_DESC(tcp_test_fail_send_netns, + "DEBUG only: target network namespace for fatal send injection"); +module_param_named(tcp_test_fail_send_ifindex, wg_tcp_test_fail_send_ifindex, + uint, 0600); +MODULE_PARM_DESC(tcp_test_fail_send_ifindex, + "DEBUG only: target WireGuard ifindex for fatal send injection"); +module_param_named(tcp_test_fail_send_local_ipv4, + wg_tcp_test_fail_send_local_ipv4, uint, 0600); +MODULE_PARM_DESC(tcp_test_fail_send_local_ipv4, + "DEBUG only: target local IPv4 address for fatal send injection"); +module_param_named(tcp_test_fail_send_source_port, + wg_tcp_test_fail_send_source_port, uint, 0600); +MODULE_PARM_DESC(tcp_test_fail_send_source_port, + "DEBUG only: target local TCP source port for fatal send injection"); +module_param_named(tcp_test_fail_send_remote_ipv4, + wg_tcp_test_fail_send_remote_ipv4, uint, 0600); +MODULE_PARM_DESC(tcp_test_fail_send_remote_ipv4, + "DEBUG only: target remote IPv4 address for fatal send injection"); +module_param_named(tcp_test_fail_send_remote_port, + wg_tcp_test_fail_send_remote_port, uint, 0600); +MODULE_PARM_DESC(tcp_test_fail_send_remote_port, + "DEBUG only: target remote TCP port for fatal send injection"); +module_param_named(tcp_test_fail_next_send, wg_tcp_test_fail_next_send, + uint, 0600); +MODULE_PARM_DESC(tcp_test_fail_next_send, + "DEBUG only: arm one EPIPE on the selected TCP carrier"); + +static int wg_tcp_test_counter_get(char *buffer, + const struct kernel_param *parameter) +{ + const atomic64_t *counter = parameter->arg; + + return scnprintf(buffer, PAGE_SIZE, "%lld\n", + (long long)atomic64_read(counter)); +} + +static const struct kernel_param_ops wg_tcp_test_counter_ops = { + .get = wg_tcp_test_counter_get, +}; + +module_param_cb(tcp_test_short_writes, &wg_tcp_test_counter_ops, + &wg_tcp_test_short_writes, 0400); +MODULE_PARM_DESC(tcp_test_short_writes, + "DEBUG only: number of observed partial TCP writes"); +module_param_cb(tcp_test_injected_prefixes, &wg_tcp_test_counter_ops, + &wg_tcp_test_injected_prefixes, 0400); +MODULE_PARM_DESC(tcp_test_injected_prefixes, + "DEBUG only: number of TCP records prefixed with test garbage"); +module_param_cb(tcp_test_resyncs, &wg_tcp_test_counter_ops, + &wg_tcp_test_resyncs, 0400); +MODULE_PARM_DESC(tcp_test_resyncs, + "DEBUG only: number of successful TCP parser resynchronizations"); +module_param_cb(tcp_test_queue_drops, &wg_tcp_test_counter_ops, + &wg_tcp_test_queue_drops, 0400); +MODULE_PARM_DESC(tcp_test_queue_drops, + "DEBUG only: number of frames rejected by TCP queue pressure"); +module_param_cb(tcp_test_fatal_send_errors, &wg_tcp_test_counter_ops, + &wg_tcp_test_fatal_send_errors, 0400); +MODULE_PARM_DESC(tcp_test_fatal_send_errors, + "DEBUG only: number of terminal TCP frame send failures"); + +static size_t wg_tcp_test_send_len(size_t frame_len) +{ + unsigned int configured = READ_ONCE(wg_tcp_test_max_send_bytes); + + return configured ? min_t(size_t, configured, frame_len) : frame_len; +} + +static size_t wg_tcp_test_prefix_len(void) +{ + return min_t(size_t, READ_ONCE(wg_tcp_test_garbage_prefix_bytes), + WG_TCP_TEST_MAX_GARBAGE_PREFIX); +} + +static unsigned int wg_tcp_test_effective_queue_limit(void) +{ + unsigned int configured = READ_ONCE(wg_tcp_test_queue_limit); + + return configured && configured < MAX_QUEUED_PACKETS ? + configured : MAX_QUEUED_PACKETS; +} + +static unsigned int wg_tcp_test_take_write_delay_ms(void) +{ + return min_t(unsigned int, xchg(&wg_tcp_test_write_delay_ms, 0U), + WG_TCP_TEST_MAX_WRITE_DELAY_MS); +} + +static bool wg_tcp_test_take_fatal_send(struct wg_peer *peer, + struct socket *socket) +{ + struct sock *sk; + + if (READ_ONCE(wg_tcp_test_fail_next_send) != 1U || !peer || + !peer->device || !peer->device->dev || !socket || !socket->sk) + return false; + sk = socket->sk; + if (READ_ONCE(wg_tcp_test_fail_send_netns) != sock_net(sk)->ns.inum || + READ_ONCE(wg_tcp_test_fail_send_ifindex) != + (unsigned int)peer->device->dev->ifindex || + sk->sk_family != AF_INET || + READ_ONCE(wg_tcp_test_fail_send_local_ipv4) != + ntohl(inet_sk(sk)->inet_saddr) || + READ_ONCE(wg_tcp_test_fail_send_source_port) != + ntohs(inet_sk(sk)->inet_sport) || + READ_ONCE(wg_tcp_test_fail_send_remote_ipv4) != + ntohl(inet_sk(sk)->inet_daddr) || + READ_ONCE(wg_tcp_test_fail_send_remote_port) != + ntohs(inet_sk(sk)->inet_dport)) + return false; + return cmpxchg(&wg_tcp_test_fail_next_send, 1U, 0U) == 1U; +} +#else +static size_t wg_tcp_test_send_len(size_t frame_len) +{ + return frame_len; +} + +static size_t wg_tcp_test_prefix_len(void) +{ + return 0; +} + +static unsigned int wg_tcp_test_effective_queue_limit(void) +{ + return MAX_QUEUED_PACKETS; +} + +static unsigned int wg_tcp_test_take_write_delay_ms(void) +{ + return 0; +} + +static bool wg_tcp_test_take_fatal_send(struct wg_peer *peer, + struct socket *socket) +{ + (void)peer; + (void)socket; + return false; +} +#endif + +#define WG_TCP_MAX_PENDING_CONNECTIONS 128 +#define WG_TCP_MAX_TRACKED_CONNECTIONS 1024 +#define WG_TCP_AUTH_IDLE_TIMEOUT_MS 5000 +#define WG_TCP_AUTH_MAX_LIFETIME_MS 30000 +#define WG_TCP_CLEANUP_INTERVAL_MS 1000 +#define WG_TCP_MAX_PENDING_PER_SOURCE 8 +#define WG_TCP_ACCEPT_BURST 32 +#define WG_TCP_ACCEPT_WINDOW HZ + +static void wg_finish_tcp_connection_init(struct wg_device *wg, + struct socket *socket); +static void wg_destroy_temp_peer(struct wg_peer *peer); +static void +wg_destroy_tcp_connection_entry(struct wg_device *wg, + struct wg_tcp_socket_list_entry *entry); +static void wg_touch_tcp_connection(struct wg_peer *peer); + +static int wg_setup_tcp_socket_callbacks(struct wg_peer *peer, + struct socket *socket, bool inbound); +static int wg_reset_tcp_socket_callbacks(struct wg_peer *peer, + struct socket *socket, bool inbound); +static int wg_reset_exact_tcp_socket_callbacks(struct wg_peer *peer, + struct socket *socket); +void wg_get_endpoint_from_socket(struct socket *epsocket, struct endpoint *ep); +static __be16 wg_header_checksum(const struct wg_tcp_encap_header *hdr); +static void wg_tcp_mark_connection_authenticated(struct wg_device *wg, + u64 connection_id); +static bool wg_tcp_promote_authenticated_carrier(struct wg_peer *peer, + u64 connection_id); + +static struct sk_buff *wg_tcp_build_frame(const struct sk_buff *payload) +{ + struct wg_tcp_encap_header encap_header = { + .type = WG_TCP_RECORD_DATA, + .flags = 0 + }; + struct wg_tcp_frag_header frag_header; + struct sk_buff *frame; + bool fragmented = PACKET_CB(payload)->frag_off != 0; + size_t header_len = WG_TCP_ENCAP_HDR_LEN; + size_t prefix_len = wg_tcp_test_prefix_len(); + size_t total_len; + + if (payload->len < MESSAGE_MINIMUM_LENGTH) + return ERR_PTR(-EINVAL); + if (fragmented) { + encap_header.flags = WG_TCP_FRAG_FLAG; + frag_header.id = PACKET_CB(payload)->frag_id; + frag_header.frag_off = PACKET_CB(payload)->frag_off; + header_len += WG_TCP_FRAG_HDR_LEN; + } + if (payload->len > WG_MAX_PACKET_SIZE - header_len) + return ERR_PTR(-EMSGSIZE); + + total_len = header_len + payload->len; + encap_header.length = htonl(total_len); + encap_header.checksum = wg_header_checksum(&encap_header); + frame = alloc_skb(prefix_len + total_len, GFP_ATOMIC); + if (!frame) + return ERR_PTR(-ENOMEM); + + if (prefix_len) { + /* Any candidate beginning in this prefix has 0xa5 as the high byte + * of its network-order length, so it cannot pass the bounded header + * validator even when the candidate overlaps the real header. + */ + memset(skb_put(frame, prefix_len), 0xa5, prefix_len); +#if defined(DEBUG) && defined(WG_TCP_FAULT_INJECTION) + atomic64_inc(&wg_tcp_test_injected_prefixes); +#endif + } + skb_put_data(frame, &encap_header, WG_TCP_ENCAP_HDR_LEN); + if (fragmented) + skb_put_data(frame, &frag_header, WG_TCP_FRAG_HDR_LEN); + if (skb_copy_bits(payload, 0, skb_put(frame, payload->len), + payload->len)) { + kfree_skb(frame); + return ERR_PTR(-EINVAL); + } + return frame; +} + +/* Queue the serial writer while holding the same lifetime lock used to claim + * socket removal. queue_work() stays inside tcp_lock so teardown cannot set a + * removal flag, finish cancel_work_sync(), and release the socket before the + * newly claimed work is visible to the workqueue. + */ +static void wg_tcp_schedule_write_locked(struct wg_peer *peer) +{ + lockdep_assert_held(&peer->tcp_lock); + spin_lock(&peer->tcp_write_lock); + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + !peer->tcp_outbound_remove_scheduled && + !peer->tcp_inbound_remove_scheduled && peer->peer_socket && + peer->tcp_established && peer->tcp_write_wq && + !peer->tcp_write_worker_scheduled) { + peer->tcp_write_worker_scheduled = true; + queue_work(peer->tcp_write_wq, &peer->tcp_write_work); + } + spin_unlock(&peer->tcp_write_lock); + +} + +static void wg_tcp_schedule_write(struct wg_peer *peer) +{ + if (!peer || IS_ERR(peer)) + return; + + spin_lock_bh(&peer->tcp_lock); + wg_tcp_schedule_write_locked(peer); + spin_unlock_bh(&peer->tcp_lock); +} + +static int wg_tcp_enqueue_frame(struct wg_peer *peer, struct sk_buff *frame) +{ + unsigned int queue_limit = wg_tcp_test_effective_queue_limit(); + int ret = 0; + + /* The queue and writer claim share the peer lifetime lock. Once stop sets + * tcp_stopping, no producer can append a frame or queue work after the + * final cancellation pass. + */ + spin_lock_bh(&peer->tcp_lock); + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP || + !peer->peer_socket || !peer->tcp_established || + peer->tcp_outbound_remove_scheduled || + peer->tcp_inbound_remove_scheduled) { + ret = -ENOTCONN; + goto unlock_tcp; + } + + spin_lock(&peer->send_queue_lock); + /* Preserve stream order. In particular, the head can contain the + * unconsumed suffix of a frame whose prefix is already on the wire. + */ + if (skb_queue_len(&peer->send_queue) >= queue_limit) { + ret = -ENOBUFS; +#if defined(DEBUG) && defined(WG_TCP_FAULT_INJECTION) + atomic64_inc(&wg_tcp_test_queue_drops); +#endif + } else { + __skb_queue_tail(&peer->send_queue, frame); + } + spin_unlock(&peer->send_queue_lock); + if (!ret) + wg_tcp_schedule_write_locked(peer); + +unlock_tcp: + spin_unlock_bh(&peer->tcp_lock); + + if (ret) { + kfree_skb(frame); + return ret; + } + return 0; +} + +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) +{ + wg_dbg("Entering function wg_socket_send_skb_to_peer\n"); + size_t skb_len; + int ret = -EAFNOSUPPORT; + bool tcp_connected = false; + + if (unlikely(!peer) || unlikely(IS_ERR(peer))){ + ret = -EINVAL; + goto out; + } + if (unlikely(!skb)){ + ret = -ENOMEM; + goto out; + } + skb_len = skb->len; + + print_peer_socket_info(peer); + + if (peer->device->transport == WG_TRANSPORT_TCP) { + spin_lock_bh(&peer->tcp_lock); + tcp_connected = !READ_ONCE(peer->is_dead) && + !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + peer->peer_socket && peer->tcp_established && + !peer->tcp_outbound_remove_scheduled && + !peer->tcp_inbound_remove_scheduled; + spin_unlock_bh(&peer->tcp_lock); + if (likely(tcp_connected)) { + struct sk_buff *frame = wg_tcp_build_frame(skb); + + kfree_skb(skb); + if (IS_ERR(frame)) + ret = PTR_ERR(frame); + else + ret = wg_tcp_enqueue_frame(peer, frame); + } else { + ret = -ENOTCONN; + spin_lock_bh(&peer->tcp_lock); + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + peer->device->transport == WG_TRANSPORT_TCP && + peer->peer_endpoint_set && !peer->tcp_retry_scheduled && + !peer->tcp_outbound_remove_scheduled) { + peer->tcp_retry_scheduled = true; + mod_delayed_work(system_wq, &peer->tcp_retry_work, 0); + } + spin_unlock_bh(&peer->tcp_lock); + net_dbg_ratelimited("%s: TCP peer %llu is reconnecting\n", + peer->device->dev->name, + peer->internal_id); + kfree_skb(skb); + } + } else { + read_lock_bh(&peer->endpoint_lock); + ret = wg_socket_send_skb_to_endpoint(peer->device, skb, + &peer->endpoint, ds, + &peer->endpoint_cache); + read_unlock_bh(&peer->endpoint_lock); + } + if (ret == 0) + peer->tx_bytes += skb_len; +out: + wg_dbg("Exiting function wg_socket_send_skb_to_peer\n"); + return ret; + +} +static bool wg_tcp_dial_target_eq(const struct endpoint *a, + const struct endpoint *b) +{ + if (a->addr.sa_family != b->addr.sa_family) + return false; + if (a->addr.sa_family == AF_INET) + return a->addr4.sin_port == b->addr4.sin_port && + a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr; +#if IS_ENABLED(CONFIG_IPV6) + if (a->addr.sa_family == AF_INET6) + return a->addr6.sin6_port == b->addr6.sin6_port && + ipv6_addr_equal(&a->addr6.sin6_addr, + &b->addr6.sin6_addr) && + a->addr6.sin6_scope_id == b->addr6.sin6_scope_id; +#endif + return false; +} + +static void wg_release_peer_tcp_connection(struct wg_peer *peer); + +static void wg_tcp_peer_request_reconnect_after(struct wg_peer *peer, + unsigned long delay) +{ + bool queue_outbound_remove = false; + + if (!peer || IS_ERR(peer) || !peer->device) + return; + + /* This helper is callable from authenticated receive/NAPI context. Claim + * and queue the process-context removal owner without shutting down the + * socket inline. Queueing under tcp_lock closes the race with peer stop + * setting its barrier and draining this work item. + */ + spin_lock_bh(&peer->tcp_lock); + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP || + !netif_running(peer->device->dev) || !peer->peer_endpoint_set) { + spin_unlock_bh(&peer->tcp_lock); + return; + } + peer->tcp_reconnect_requested = true; + if (!peer->tcp_connecting && !peer->tcp_outbound_remove_scheduled) { + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = peer->outbound_socket; + queue_outbound_remove = true; + } + if (queue_outbound_remove) + mod_delayed_work(system_wq, &peer->tcp_outbound_remove_work, + delay); + spin_unlock_bh(&peer->tcp_lock); +} + +void wg_tcp_peer_request_reconnect(struct wg_peer *peer) +{ + wg_tcp_peer_request_reconnect_after(peer, 0); +} + +static void wg_socket_set_peer_endpoint_internal(struct wg_peer *peer, + const struct endpoint *endpoint, + bool configured) +{ + bool tcp_target_changed = false; + + wg_dbg("Entering function wg_socket_set_peer_endpoint peer=%px\n", peer); + if (unlikely(!peer) || unlikely(IS_ERR(peer))){ + goto out; + } + + /* First we check unlocked, in order to optimize, since it's pretty rare + * that an endpoint will change. If we happen to be mid-write, and two + * CPUs wind up writing the same thing or something slightly different, + * it doesn't really matter much either. + */ + if (endpoint_eq(endpoint, &peer->endpoint) && + (!configured || peer->device->transport != WG_TRANSPORT_TCP || + (peer->peer_endpoint_set && + endpoint_eq(endpoint, &peer->peer_endpoint)))) { + wg_dbg("Exiting function wg_socket_set_peer_endpoint (no change in endpoint)\n"); + return; + } + + print_peer_socket_info(peer); + + write_lock_bh(&peer->endpoint_lock); + if (endpoint->addr.sa_family == AF_INET) { + wg_dbg("Setting endpoint address: %pI4:%d\n", + &endpoint->addr4.sin_addr, + ntohs(endpoint->addr4.sin_port)); + peer->endpoint.addr4 = endpoint->addr4; + peer->endpoint.src4 = endpoint->src4; + peer->endpoint.src_if4 = endpoint->src_if4; + } else if (IS_ENABLED(CONFIG_IPV6) && endpoint->addr.sa_family == AF_INET6) { + wg_dbg("Setting endpoint address: [%pI6]:%d\n", + &endpoint->addr6.sin6_addr, + ntohs(endpoint->addr6.sin6_port)); + peer->endpoint.addr6 = endpoint->addr6; + peer->endpoint.src6 = endpoint->src6; + } else { + write_unlock_bh(&peer->endpoint_lock); + goto out; + } + dst_cache_reset(&peer->endpoint_cache); + if (peer->device->transport == WG_TRANSPORT_TCP) { + peer->tcp_reply_endpoint = peer->endpoint; + if (configured) { + tcp_target_changed = peer->peer_endpoint_set && + !endpoint_eq(&peer->peer_endpoint, &peer->endpoint); + peer->peer_endpoint = peer->endpoint; + peer->tcp_peer_listen_port = + peer->endpoint.addr.sa_family == AF_INET ? + peer->endpoint.addr4.sin_port : + peer->endpoint.addr6.sin6_port; + peer->peer_endpoint_set = true; + } + } + write_unlock_bh(&peer->endpoint_lock); + if (peer->device->transport != WG_TRANSPORT_TCP || !configured) + goto out; + + wg_dbg("Peer Endpoint:\n"); + log_wireguard_endpoint(&peer->endpoint); + wg_dbg("TCP Reply Endpoint:\n"); + log_wireguard_endpoint(&peer->tcp_reply_endpoint); + + /* A configured target change owns the reconnect request. Mark removal + * before shutdown so the state callback cannot race us to queue a second + * owner for the same socket. The removal worker releases the old stream + * before it arms an immediate reconnect. + */ + if (tcp_target_changed) { + wg_tcp_peer_request_reconnect(peer); + } else if (netif_running(peer->device->dev) && + !peer->tcp_established) { + wg_tcp_connect(peer); + } + +out: + wg_dbg("Exiting function wg_socket_set_peer_endpoint\n"); +} + +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint) +{ + wg_socket_set_peer_endpoint_internal(peer, endpoint, false); +} + +void wg_socket_set_peer_endpoint_configured(struct wg_peer *peer, + const struct endpoint *endpoint) +{ + wg_socket_set_peer_endpoint_internal(peer, endpoint, true); +} + +void wg_socket_set_peer_endpoint_authenticated(struct wg_peer *peer, + const struct endpoint *endpoint, + u64 connection_id) +{ + struct endpoint target; + bool target_changed = false; + + if (unlikely(!peer) || unlikely(IS_ERR(peer)) || unlikely(!endpoint)) + return; + + /* Keep the complete live tuple as observed state. For TCP, only the + * authenticated remote address may refresh the future dial target: an + * accepted socket's remote port is normally ephemeral and must never + * replace the operator-configured peer listen port. + */ + wg_socket_set_peer_endpoint(peer, endpoint); + if (peer->device->transport != WG_TRANSPORT_TCP) + return; + if (connection_id) { + wg_tcp_mark_connection_authenticated(peer->device, connection_id); + spin_lock_bh(&peer->tcp_lock); + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + connection_id > peer->tcp_promotion_connection_id) { + peer->tcp_promotion_connection_id = connection_id; + if (!peer->tcp_promotion_worker_scheduled) { + peer->tcp_promotion_worker_scheduled = true; + queue_work(system_wq, &peer->tcp_promotion_work); + } + } + spin_unlock_bh(&peer->tcp_lock); + } + + write_lock_bh(&peer->endpoint_lock); + /* Only authenticated accepted carriers have a nonzero, device-monotonic + * ID. Advancing the generation even when the address is unchanged keeps + * an older retained stream from reverting a later roaming observation. + */ + if (!peer->peer_endpoint_set || !connection_id || + connection_id < peer->tcp_roaming_connection_id) + goto out; + target = *endpoint; + if (target.addr.sa_family == AF_INET) { + target.addr4.sin_port = peer->tcp_peer_listen_port; + target.src4.s_addr = 0; + target.src_if4 = 0; + } else if (IS_ENABLED(CONFIG_IPV6) && + target.addr.sa_family == AF_INET6) { + target.addr6.sin6_port = peer->tcp_peer_listen_port; + memset(&target.src6, 0, sizeof(target.src6)); + } else { + goto out; + } + peer->tcp_roaming_connection_id = connection_id; + if (!wg_tcp_dial_target_eq(&peer->peer_endpoint, &target)) { + peer->peer_endpoint = target; + dst_cache_reset(&peer->endpoint_cache); + target_changed = true; + } +out: + write_unlock_bh(&peer->endpoint_lock); + if (target_changed && !peer->temp_peer) + wg_tcp_peer_request_reconnect_after(peer, + msecs_to_jiffies(100)); +} + +void wg_socket_set_peer_endpoint_authenticated_from_skb( + struct wg_peer *peer, const struct sk_buff *skb) +{ + struct endpoint endpoint; + + if (likely(!wg_socket_endpoint_from_skb(&endpoint, skb))) + wg_socket_set_peer_endpoint_authenticated( + peer, &endpoint, PACKET_CB(skb)->tcp_connection_id); +} + +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb) +{ + wg_dbg("Entering function wg_socket_set_peer_endpoint_from_skb peer=%px\n", peer); + struct endpoint endpoint; + + if (unlikely(!peer) || unlikely(IS_ERR(peer))){ + goto out; + } + + if (!wg_socket_endpoint_from_skb(&endpoint, skb)) + wg_socket_set_peer_endpoint(peer, &endpoint); + log_wireguard_endpoint(&peer->endpoint); + print_peer_socket_info(peer); +out: + wg_dbg("Exiting function wg_socket_set_peer_endpoint_from_skb\n"); +} + +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) +{ + wg_dbg("Entering function wg_socket_clear_peer_endpoint_src\n"); + write_lock_bh(&peer->endpoint_lock); + memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); + dst_cache_reset_now(&peer->endpoint_cache); + write_unlock_bh(&peer->endpoint_lock); + wg_dbg("Exiting function wg_socket_clear_peer_endpoint_src\n"); +} + +static int wg_receive(struct sock *sk, struct sk_buff *skb) +{ + wg_dbg("Entering function wg_receive\n"); + struct wg_device *wg; + struct wg_socket_data *socket_data = NULL; + + if (unlikely(!sk)) + goto err; + if (sk->sk_protocol == IPPROTO_TCP) { + socket_data = READ_ONCE(sk->sk_user_data); + + if (unlikely(!socket_data)) + goto err; + wg = socket_data->device; + } else { + wg = READ_ONCE(sk->sk_user_data); + } + if (unlikely(!wg)) + goto err; + PACKET_CB(skb)->outer_ipproto = sk->sk_protocol; + PACKET_CB(skb)->tcp_connection_id = + socket_data && socket_data->peer && socket_data->peer->temp_peer ? + socket_data->peer->tcp_connection_id : 0; + skb_mark_not_on_list(skb); + wg_packet_receive(wg, skb); + wg_dbg("Exiting function wg_receive\n"); + return 0; + +err: + kfree_skb(skb); + wg_dbg("Exiting function wg_receive with error.\n"); + return 0; +} +static int wg_set_socket_timeouts(struct socket *sock, unsigned long snd_timeout, + unsigned long rcv_timeout) +{ + wg_dbg("Entering function wg_set_socket_timeouts\n"); + if (!sock || !sock->sk) { + pr_err("Invalid socket or sock is NULL\n"); + return -EINVAL; + } + + struct sock *sk = sock->sk; + + sk->sk_sndtimeo = snd_timeout*30; + sk->sk_rcvtimeo = rcv_timeout*30; + + wg_dbg("Exiting function wg_set_socket_timeouts\n"); + return 0; +} + +static bool wg_sockaddrs_match(const struct sockaddr *a, + const struct sockaddr *b) +{ + if (!a || !b || a->sa_family != b->sa_family) + return false; + + if (a->sa_family == AF_INET) { + const struct sockaddr_in *a4 = (const struct sockaddr_in *)a; + const struct sockaddr_in *b4 = (const struct sockaddr_in *)b; + + return a4->sin_port == b4->sin_port && + a4->sin_addr.s_addr == b4->sin_addr.s_addr; + } +#if IS_ENABLED(CONFIG_IPV6) + if (a->sa_family == AF_INET6) { + const struct sockaddr_in6 *a6 = (const struct sockaddr_in6 *)a; + const struct sockaddr_in6 *b6 = (const struct sockaddr_in6 *)b; + const bool a_link_local = + ipv6_addr_type(&a6->sin6_addr) & IPV6_ADDR_LINKLOCAL; + const bool b_link_local = + ipv6_addr_type(&b6->sin6_addr) & IPV6_ADDR_LINKLOCAL; + + return a6->sin6_port == b6->sin6_port && + ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) && + (!a_link_local || !b_link_local || + a6->sin6_scope_id == b6->sin6_scope_id); + } +#endif + return false; +} + +static bool wg_sockaddrs_same_host(const struct sockaddr *a, + const struct sockaddr *b) +{ + if (!a || !b || a->sa_family != b->sa_family) + return false; + + if (a->sa_family == AF_INET) { + const struct sockaddr_in *a4 = (const struct sockaddr_in *)a; + const struct sockaddr_in *b4 = (const struct sockaddr_in *)b; + + return a4->sin_addr.s_addr == b4->sin_addr.s_addr; + } +#if IS_ENABLED(CONFIG_IPV6) + if (a->sa_family == AF_INET6) { + const struct sockaddr_in6 *a6 = (const struct sockaddr_in6 *)a; + const struct sockaddr_in6 *b6 = (const struct sockaddr_in6 *)b; + const bool link_local = + ipv6_addr_type(&a6->sin6_addr) & IPV6_ADDR_LINKLOCAL; + + return ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) && + (!link_local || a6->sin6_scope_id == b6->sin6_scope_id); + } +#endif + return false; +} + +static bool wg_sockaddr_length_valid(const struct sockaddr *addr, int length) +{ + if (!addr) + return false; + if (addr->sa_family == AF_INET) + return length >= sizeof(struct sockaddr_in); +#if IS_ENABLED(CONFIG_IPV6) + if (addr->sa_family == AF_INET6) + return length >= sizeof(struct sockaddr_in6); +#endif + return false; +} + +static bool wg_endpoints_match(const struct endpoint *a, + const struct endpoint *b) +{ + return a && b && wg_sockaddrs_match(&a->addr, &b->addr); +} + +static bool wg_tcp_accept_source_matches( + const struct wg_tcp_accept_source *source, const struct sockaddr *addr) +{ + if (!source || !addr || source->family != addr->sa_family) + return false; + if (addr->sa_family == AF_INET) + return source->address.addr4 == + ((const struct sockaddr_in *)addr)->sin_addr.s_addr; +#if IS_ENABLED(CONFIG_IPV6) + if (addr->sa_family == AF_INET6) { + const struct sockaddr_in6 *addr6 = + (const struct sockaddr_in6 *)addr; + const bool link_local = + ipv6_addr_type(&addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL; + + return ipv6_addr_equal(&source->address.addr6, + &addr6->sin6_addr) && + (!link_local || source->scope_id == addr6->sin6_scope_id); + } +#endif + return false; +} + +static void wg_tcp_accept_source_set(struct wg_tcp_accept_source *source, + const struct sockaddr *addr, + unsigned long now) +{ + memset(source, 0, sizeof(*source)); + source->family = addr->sa_family; + if (addr->sa_family == AF_INET) { + source->address.addr4 = + ((const struct sockaddr_in *)addr)->sin_addr.s_addr; +#if IS_ENABLED(CONFIG_IPV6) + } else if (addr->sa_family == AF_INET6) { + const struct sockaddr_in6 *addr6 = + (const struct sockaddr_in6 *)addr; + + source->address.addr6 = addr6->sin6_addr; + if (ipv6_addr_type(&addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL) + source->scope_id = addr6->sin6_scope_id; +#endif + } + source->window_started = now; + source->last_seen = now; + source->accepts = 1; +} + +/* Bound rapid provisional-peer creation without allocating attacker-owned + * tracking state. The fixed table is deliberately lossy under a many-source + * flood; the device-wide pending cap remains the final backstop. + */ +static bool wg_tcp_accept_rate_allow(struct wg_device *wg, + const struct sockaddr *addr) +{ + struct wg_tcp_accept_source *empty = NULL, *oldest = NULL, *source = NULL; + const unsigned long now = jiffies; + bool allowed = true; + unsigned int i; + + if (!wg || !addr || (addr->sa_family != AF_INET && + addr->sa_family != AF_INET6)) + return false; + + spin_lock_bh(&wg->tcp_accept_lock); + for (i = 0; i < WG_TCP_ACCEPT_SOURCE_SLOTS; ++i) { + struct wg_tcp_accept_source *candidate = + &wg->tcp_accept_sources[i]; + + if (!candidate->family) { + if (!empty) + empty = candidate; + continue; + } + if (wg_tcp_accept_source_matches(candidate, addr)) { + source = candidate; + break; + } + if (!oldest || time_before(candidate->last_seen, + oldest->last_seen)) + oldest = candidate; + } + + if (!source) { + source = empty ? empty : oldest; + wg_tcp_accept_source_set(source, addr, now); + } else if (time_after_eq(now, source->window_started + + WG_TCP_ACCEPT_WINDOW)) { + wg_tcp_accept_source_set(source, addr, now); + } else if (source->accepts >= WG_TCP_ACCEPT_BURST) { + source->last_seen = now; + allowed = false; + } else { + ++source->accepts; + source->last_seen = now; + } + spin_unlock_bh(&wg->tcp_accept_lock); + return allowed; +} + +static unsigned int +wg_tcp_pending_from_source_locked(struct wg_device *wg, + const struct sockaddr *addr) +{ + struct wg_tcp_socket_list_entry *entry; + unsigned int count = 0; + + list_for_each_entry(entry, &wg->tcp_connection_list, tcp_connection_ll) { + if (entry->admission_counted && + wg_sockaddrs_same_host( + addr, (const struct sockaddr *)&entry->src_addr)) + ++count; + } + return count; +} + +static bool wg_tcp_source_at_capacity(struct wg_device *wg, + const struct sockaddr *addr) +{ + bool at_capacity; + + spin_lock_bh(&wg->tcp_connection_list_lock); + at_capacity = wg_tcp_pending_from_source_locked(wg, addr) >= + WG_TCP_MAX_PENDING_PER_SOURCE; + spin_unlock_bh(&wg->tcp_connection_list_lock); + return at_capacity; +} + +static void +wg_tcp_release_admission_locked(struct wg_device *wg, + struct wg_tcp_socket_list_entry *entry) +{ + lockdep_assert_held(&wg->tcp_connection_list_lock); + if (!entry->admission_counted) + return; + entry->admission_counted = false; + if (WARN_ON_ONCE(!wg->tcp_pending_connections)) + return; + --wg->tcp_pending_connections; +} + +static void wg_tcp_mark_connection_authenticated(struct wg_device *wg, + u64 connection_id) +{ + struct wg_tcp_socket_list_entry *entry; + + if (!wg || !connection_id) + return; + spin_lock_bh(&wg->tcp_connection_list_lock); + list_for_each_entry(entry, &wg->tcp_connection_list, tcp_connection_ll) { + if (entry->connection_id != connection_id) + continue; + entry->authenticated = true; + wg_tcp_release_admission_locked(wg, entry); + entry->timestamp = ktime_get(); + break; + } + spin_unlock_bh(&wg->tcp_connection_list_lock); +} + +void wg_tcp_set_device_mark(struct wg_device *wg, u32 mark) +{ + struct wg_tcp_socket_list_entry *entry; + + if (!wg) + return; + if (wg->tcp_listen_socket4 && wg->tcp_listen_socket4->sk) + WRITE_ONCE(wg->tcp_listen_socket4->sk->sk_mark, mark); +#if IS_ENABLED(CONFIG_IPV6) + if (wg->tcp_listen_socket6 && wg->tcp_listen_socket6->sk) + WRITE_ONCE(wg->tcp_listen_socket6->sk->sk_mark, mark); +#endif + + /* Accepted carriers stay device-owned until cleanup or promotion. The + * list lock keeps each socket alive while its mark is refreshed. + */ + spin_lock_bh(&wg->tcp_connection_list_lock); + list_for_each_entry(entry, &wg->tcp_connection_list, tcp_connection_ll) { + if (entry->tcp_socket && entry->tcp_socket->sk) + WRITE_ONCE(entry->tcp_socket->sk->sk_mark, mark); + } + spin_unlock_bh(&wg->tcp_connection_list_lock); +} + +void wg_free_peer_socket_data(struct wg_peer *peer); + +void wg_free_peer_socket_data(struct wg_peer *peer) +{ + if (peer && !IS_ERR(peer)) + if (peer->peer_socket) + if (peer->peer_socket->sk) + if (peer->peer_socket->sk->sk_user_data) + kfree(peer->peer_socket->sk->sk_user_data); +} + +static int wg_release_peer_socket_locked(struct wg_peer *peer, + struct socket *socket) +{ + struct sk_buff *partial = NULL; + bool inbound, outbound; + bool active; + + if (!peer || IS_ERR(peer) || !socket) + return -EINVAL; + lockdep_assert_held(&peer->tcp_socket_lock); + + spin_lock_bh(&peer->tcp_lock); + inbound = peer->inbound_socket == socket; + outbound = peer->outbound_socket == socket; + if (!inbound && !outbound) { + spin_unlock_bh(&peer->tcp_lock); + return -ESTALE; + } + if ((inbound && (peer->tcp_inbound_callbacks_set || + peer->tcp_inbound_socket_data)) || + (outbound && (peer->tcp_outbound_callbacks_set || + peer->tcp_outbound_socket_data))) { + spin_unlock_bh(&peer->tcp_lock); + return -EBUSY; + } + active = peer->peer_socket == socket; + if (active) { + peer->peer_socket = NULL; + partial = peer->partial_skb; + peer->partial_skb = NULL; + peer->received_len = 0; + peer->expected_len = 0; + peer->tcp_pending = false; + peer->tcp_retry_scheduled = false; + } + if (inbound) { + peer->inbound_socket = NULL; + peer->inbound_connected = false; + peer->inbound_timestamp = ktime_set(0, 0); + } + if (outbound) { + peer->outbound_socket = NULL; + peer->outbound_connected = false; + peer->outbound_timestamp = ktime_set(0, 0); + } + if (active || (!peer->inbound_connected && !peer->outbound_connected)) + peer->tcp_established = false; + spin_unlock_bh(&peer->tcp_lock); + + if (partial) + kfree_skb(partial); + if (active) { + spin_lock_bh(&peer->send_queue_lock); + __skb_queue_purge(&peer->send_queue); + spin_unlock_bh(&peer->send_queue_lock); + } + kernel_sock_shutdown(socket, SHUT_RDWR); + sock_release(socket); + return 0; +} + + + +void wg_clean_peer_socket(struct wg_peer *peer, bool release, bool destroy, bool inbound) +{ + wg_dbg("Entering function wg_clean_peer_socket peer=%px, inbound=%d\n", peer, inbound); + if (!peer || IS_ERR(peer)) { + wg_dbg("wg_clean_peer_socket: No peer or invalid peer.\n"); + goto out; + } + print_peer_socket_info(peer); + if ((inbound && peer->peer_socket == peer->inbound_socket) || + (!inbound && peer->peer_socket == peer->outbound_socket)) { + /* Cleanup partial skb buffer */ + if (peer->partial_skb) { + kfree_skb(peer->partial_skb); + peer->partial_skb = NULL; + } + + /* Cancel and flush the TCP read workqueue */ + if (peer->tcp_read_worker_scheduled) { + cancel_work_sync(&peer->tcp_read_work); + peer->tcp_read_worker_scheduled = false; + } + if (peer->tcp_read_wq && destroy) { + destroy_workqueue(peer->tcp_read_wq); + peer->tcp_read_wq = NULL; + } + + /* Cancel and flush the TCP write workqueue */ + if (peer->tcp_write_worker_scheduled) { + cancel_work_sync(&peer->tcp_write_work); + peer->tcp_write_worker_scheduled = false; + } + if (peer->tcp_write_wq && destroy) { + destroy_workqueue(peer->tcp_write_wq); + peer->tcp_write_wq = NULL; + } + + /* Clean up packet queues */ + if (!skb_queue_empty(&peer->send_queue)) + skb_queue_purge(&peer->send_queue); + + /* Reset TCP state */ + peer->received_len = 0; + peer->expected_len = 0; + peer->tcp_established = false; + peer->tcp_pending = false; + peer->tcp_retry_scheduled = false; + } + + /* Determine which socket and related resources to clean based on the 'inbound' flag */ + struct socket **socket_to_clean = inbound ? &peer->inbound_socket : &peer->outbound_socket; + bool *callbacks_set_flag = inbound ? &peer->tcp_inbound_callbacks_set : &peer->tcp_outbound_callbacks_set; + bool *connected_flag = inbound ? &peer->inbound_connected : &peer->outbound_connected; + ktime_t *timestamp = inbound ? &peer->inbound_timestamp : &peer->outbound_timestamp; + /* Cleanup socket if necessary */ + if (*socket_to_clean) { + if (peer->peer_socket == *socket_to_clean) + peer->peer_socket = NULL; + if (release) { + /* Directly free peer socket data as per wg_free_peer_socket_data logic */ + if (*socket_to_clean && (*socket_to_clean)->sk) { + if ((*socket_to_clean)->sk->sk_user_data) { + kfree((*socket_to_clean)->sk->sk_user_data); + (*socket_to_clean)->sk->sk_user_data = NULL; + } + } + kernel_sock_shutdown(*socket_to_clean, SHUT_RDWR); + sock_release(*socket_to_clean); + } + *socket_to_clean = NULL; + } + + /* Reset callbacks set flag */ + *callbacks_set_flag = false; + + /* Reset connection status and timestamp */ + *connected_flag = false; + *timestamp = 0; + +out: + print_peer_socket_info(peer); + wg_dbg("Exiting wg_clean_peer_socket\n"); +} + +void wg_tcp_peer_stop(struct wg_peer *peer) +{ + struct socket *outbound, *inbound; + struct sock *outbound_sk, *inbound_sk; + bool quarantine_peer = false; + bool teardown_failed = false; + int ret; + + if (!peer || IS_ERR(peer)) + return; + + spin_lock_bh(&peer->tcp_lock); + peer->tcp_stopping = true; + peer->tcp_reconnect_requested = false; + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = peer->outbound_socket; + peer->tcp_inbound_remove_scheduled = true; + peer->tcp_inbound_remove_socket = peer->inbound_socket; + spin_unlock_bh(&peer->tcp_lock); + + /* Removal workers own socket destruction. Drain them before reading a + * socket pointer so stop cannot race sock_release() with a callback-lock + * snapshot. The stop barrier above prevents any peer-owned work from + * being queued after these cancellation passes. + */ + cancel_delayed_work_sync(&peer->tcp_retry_work); + cancel_delayed_work_sync(&peer->tcp_outbound_remove_work); + cancel_delayed_work_sync(&peer->tcp_inbound_remove_work); + cancel_work_sync(&peer->tcp_bootstrap_work); + cancel_work_sync(&peer->tcp_promotion_work); + mutex_lock(&peer->tcp_socket_lock); + + /* A removal worker that was already running can publish its completion + * while the cancellation waits. Reassert the stop claims before socket + * snapshotting so callbacks still see a closed scheduling gate. + */ + spin_lock_bh(&peer->tcp_lock); + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = peer->outbound_socket; + peer->tcp_inbound_remove_scheduled = true; + peer->tcp_inbound_remove_socket = peer->inbound_socket; + outbound = peer->outbound_socket; + inbound = peer->inbound_socket; + spin_unlock_bh(&peer->tcp_lock); + outbound_sk = outbound ? outbound->sk : NULL; + inbound_sk = inbound ? inbound->sk : NULL; + if (outbound_sk) { + write_lock_bh(&outbound_sk->sk_callback_lock); + write_unlock_bh(&outbound_sk->sk_callback_lock); + } + if (inbound_sk && inbound_sk != outbound_sk) { + write_lock_bh(&inbound_sk->sk_callback_lock); + write_unlock_bh(&inbound_sk->sk_callback_lock); + } + + cancel_work_sync(&peer->tcp_read_work); + cancel_work_sync(&peer->tcp_write_work); + spin_lock_bh(&peer->tcp_lock); + peer->tcp_retry_scheduled = false; + spin_lock(&peer->tcp_read_lock); + peer->tcp_read_worker_scheduled = false; + spin_unlock(&peer->tcp_read_lock); + spin_lock(&peer->tcp_write_lock); + peer->tcp_write_worker_scheduled = false; + spin_unlock(&peer->tcp_write_lock); + spin_unlock_bh(&peer->tcp_lock); + + spin_lock_bh(&peer->send_queue_lock); + __skb_queue_purge(&peer->send_queue); + spin_unlock_bh(&peer->send_queue_lock); + + if (outbound) { + ret = wg_reset_exact_tcp_socket_callbacks(peer, outbound); + if (!ret) + ret = wg_release_peer_socket_locked(peer, outbound); + if (ret) + teardown_failed = true; + } + if (inbound && inbound != outbound) { + ret = wg_reset_exact_tcp_socket_callbacks(peer, inbound); + if (!ret) + ret = wg_release_peer_socket_locked(peer, inbound); + if (ret) + teardown_failed = true; + } + mutex_unlock(&peer->tcp_socket_lock); + if (WARN_ON_ONCE(teardown_failed)) { + pr_err("WireGuard: TCP peer teardown retained an owned socket\n"); + spin_lock_bh(&peer->tcp_lock); + if (!peer->tcp_teardown_quarantined) { + peer->tcp_teardown_quarantined = true; + quarantine_peer = true; + } + spin_unlock_bh(&peer->tcp_lock); + if (quarantine_peer) + wg_peer_get(peer); + } + + spin_lock_bh(&peer->tcp_lock); + if (!teardown_failed) { + peer->tcp_established = false; + peer->tcp_pending = false; + peer->tcp_connecting = false; + peer->tcp_reconnect_requested = false; + peer->inbound_connected = false; + peer->outbound_connected = false; + peer->tcp_outbound_remove_scheduled = false; + peer->tcp_outbound_remove_socket = NULL; + peer->tcp_inbound_remove_scheduled = false; + peer->tcp_inbound_remove_socket = NULL; + } + spin_unlock_bh(&peer->tcp_lock); +} + + +struct wg_peer *wg_temp_peer_create(struct wg_device *wg); +int wg_add_tcp_socket_to_list(struct wg_device *wg, + struct socket *receive_socket, + struct wg_peer *temp_peer); + + +/* Function to copy source and destination addresses from a TCP socket */ +static int copy_sock_addresses(struct socket *tcp_socket, + struct sockaddr_storage *inbound_source, + struct sockaddr_storage *inbound_dest) +{ + int local_len, remote_len; + + if (!tcp_socket || !tcp_socket->sk || !inbound_source || !inbound_dest) + return -EINVAL; + + memset(inbound_source, 0, sizeof(*inbound_source)); + memset(inbound_dest, 0, sizeof(*inbound_dest)); + local_len = kernel_getsockname(tcp_socket, + (struct sockaddr *)inbound_source); + if (local_len < 0 || + !wg_sockaddr_length_valid((struct sockaddr *)inbound_source, + local_len)) + return local_len < 0 ? local_len : -EINVAL; + remote_len = kernel_getpeername(tcp_socket, + (struct sockaddr *)inbound_dest); + if (remote_len < 0 || + !wg_sockaddr_length_valid((struct sockaddr *)inbound_dest, + remote_len)) + return remote_len < 0 ? remote_len : -EINVAL; + if (inbound_source->ss_family != inbound_dest->ss_family) + return -EAFNOSUPPORT; + + return 0; +} + +static struct wg_peer *wg_find_peer_by_endpoints(struct wg_device *wg, const struct endpoint *endpoint) +{ + struct wg_peer *peer = NULL; + struct wg_peer *matched_peer = NULL; + + if (!wg || !endpoint) { + printk(KERN_ERR "wg_find_peer_by_endpoints: Invalid arguments, wg or endpoint is NULL\n"); + return NULL; + } + + wg_dbg("Entering function wg_find_peer_by_endpoints\n"); + + rcu_read_lock(); + list_for_each_entry_rcu(peer, &wg->peer_list, peer_list) { + if (endpoint_eq(&peer->endpoint, endpoint) || + endpoint_eq(&peer->peer_endpoint, endpoint) || + endpoint_eq(&peer->tcp_reply_endpoint, endpoint)) { + matched_peer = peer; + wg_dbg("wg_find_peer_by_endpoints: Found matching peer %px\n", matched_peer); + break; + } + } + rcu_read_unlock(); + + if (!matched_peer) { + wg_dbg("wg_find_peer_by_endpoints: No matching peer found\n"); + } + + wg_dbg("Exiting function wg_find_peer_by_endpoints peer=%px\n", matched_peer); + return matched_peer; +} + + +int wg_tcp_listener_worker(struct wg_device *wg, struct socket *tcp_socket) +{ + bool found = false; + wg_dbg("Entering function wg_tcp_listener_worker\n"); + struct socket *new_peer_connection = NULL; + + if (!tcp_socket) { + pr_err("tcp_socket is NULL\n"); + return -EINVAL; + } + while (!kthread_should_stop()) { + int err; + + err = kernel_accept(tcp_socket, &new_peer_connection, 0); + if (err < 0) { + if (kthread_should_stop() || err == -EINVAL || err == -EBADF || + err == -ENOTCONN) + break; + if (err == -EAGAIN || err == -ERESTARTSYS) + continue; + pr_err("Error accepting new connection: %d\n", err); + continue; + } + + if (!new_peer_connection) { + pr_err("new_peer_connection is NULL after kernel_accept\n"); + continue; + } + wg_dbg("wg_tcp_listener_worker accepted socket: %px new_peer_connection: %px\n", tcp_socket, &new_peer_connection); + WRITE_ONCE(new_peer_connection->sk->sk_mark, wg->fwmark); + + /* FIX #4: Disable Nagle's algorithm on accepted socket to avoid + * ~200ms delayed ACK interaction that caused 1000ms RTT + */ + tcp_sock_set_nodelay(new_peer_connection->sk); + + struct wg_peer *matched_peer = NULL; + struct wg_peer *new_temp_peer = NULL; + struct endpoint new_endpoint; + struct wg_tcp_socket_list_entry *socket_iter = NULL; + struct socket *old_pending_socket = NULL; + + /* BUG FIX: reset found at the start of each iteration — + * was never reset, so after first match all subsequent + * connections incorrectly entered the 'found' branches + */ + found = false; + + memset(&new_endpoint, 0, sizeof(new_endpoint)); + err = new_peer_connection->ops->getname( + new_peer_connection, &new_endpoint.addr, 1); + if (err < 0 || + !wg_sockaddr_length_valid(&new_endpoint.addr, err)) { + pr_err("Could not read accepted TCP peer address: %d\n", err); + kernel_sock_shutdown(new_peer_connection, SHUT_RDWR); + sock_release(new_peer_connection); + new_peer_connection = NULL; + continue; + } + if (!wg_tcp_accept_rate_allow(wg, &new_endpoint.addr) || + wg_tcp_source_at_capacity(wg, &new_endpoint.addr)) { + pr_debug_ratelimited( + "%s: throttling unauthenticated TCP source %pISpc\n", + wg->dev->name, &new_endpoint.addr); + kernel_sock_shutdown(new_peer_connection, SHUT_RDWR); + sock_release(new_peer_connection); + new_peer_connection = NULL; + continue; + } + + if (!list_empty(&wg->peer_list)) { + /* + * Match the inbound connection to a configured peer + * endpoint. + */ + rcu_read_lock(); + list_for_each_entry_rcu(matched_peer, &wg->peer_list, peer_list) { + if (wg_endpoints_match(&matched_peer->endpoint, &new_endpoint)) { + /* read data if there is any available */ + found = true; + wg_dbg("wg_tcp_listener_worker matched existing endpoint\n"); + break; + } + } + /* BUG FIX: after list_for_each_entry_rcu exhaustion (no break), + * matched_peer points to the list head (bogus pointer), not NULL. + * Reset to NULL when no match was found. + */ + if (!found) + matched_peer = NULL; + rcu_read_unlock(); + } + /* FIX: Both matched and unmatched peers need temp peer creation. + * Original code dropped unmatched connections as "martians" which + * prevented first TCP handshakes (endpoint unknown before handshake). + * Now: always create a temp peer for inbound connections so the + * handshake can be processed and the peer promoted. + */ + if (!matched_peer) { + wg_dbg("wg_tcp_listener_worker no endpoint match — new inbound connection\n"); + } else { + wg_dbg("wg_tcp_listener_worker matched existing endpoint — reconnection\n"); + } + + { + /* Clean up any existing pending connection from same source */ + /* BUG FIX: reset found before second search */ + found = false; + if (!list_empty(&wg->tcp_connection_list)) { /* BUG FIX: was peer_list — wrong list */ + rcu_read_lock(); + /* check device pending connections in tcp_connection_list */ + list_for_each_entry_rcu(socket_iter, &wg->tcp_connection_list, tcp_connection_ll) { + /* + * Skip entries without the socket state + * required for comparison. + */ + if (!socket_iter) { + wg_dbg("socket_iter is NULL\n"); + continue; + } + if (!socket_iter->tcp_socket) { + wg_dbg("socket_iter->tcp_socket is NULL\n"); + continue; + } + if (!socket_iter->tcp_socket->sk) { + wg_dbg("socket_iter->tcp_socket->sk is NULL\n"); + continue; + } + + if (wg_sockaddrs_match( + &new_endpoint.addr, + (const struct sockaddr *)&socket_iter->src_addr)) { + found = true; + old_pending_socket = socket_iter->tcp_socket; + break; + } + } + rcu_read_unlock(); + } + if (found) { + wg_dbg("wg_tcp_listener_worker new connection was for an existing peer\n"); + wg_remove_from_tcp_connection_list(wg, old_pending_socket); + } + + /* + * Queue a provisional roaming connection for + * authentication. + */ + + new_temp_peer = wg_temp_peer_create(wg); + wg_dbg("wg_tcp_listener_worker created temp peer for inbound new connection temp_peer=%px\n", new_temp_peer); + if (!IS_ERR(new_temp_peer) && new_temp_peer) { + mutex_lock(&new_temp_peer->tcp_socket_lock); + new_temp_peer->peer_socket = new_peer_connection; + new_temp_peer->inbound_socket = new_peer_connection; + + wg_get_endpoint_from_socket(new_peer_connection, &new_temp_peer->tcp_reply_endpoint); + new_temp_peer->endpoint = new_temp_peer->tcp_reply_endpoint; + + new_temp_peer->tcp_established = true; + new_temp_peer->inbound_connected = true; + new_temp_peer->inbound_timestamp = ktime_get(); + new_temp_peer->clean_inbound = false; + new_temp_peer->tcp_inbound_callbacks_set = false; + copy_sock_addresses(new_peer_connection, &new_temp_peer->inbound_source, &new_temp_peer->inbound_dest); + wg_dbg("new_temp_peer Peer endpoint:"); + log_wireguard_endpoint(&new_temp_peer->endpoint); + /* This socket's remote port is an observed ephemeral source + * port. A provisional peer has no authenticated dial target. + */ + new_temp_peer->peer_endpoint_set = false; + + err = wg_setup_tcp_socket_callbacks( + new_temp_peer, new_peer_connection, true); + if (err) { + mutex_unlock(&new_temp_peer->tcp_socket_lock); + wg_destroy_temp_peer(new_temp_peer); + continue; + } + if (wg_add_tcp_socket_to_list(wg, new_peer_connection, + new_temp_peer)) { + mutex_unlock(&new_temp_peer->tcp_socket_lock); + wg_destroy_temp_peer(new_temp_peer); + continue; + } + if (!skb_queue_empty(&new_peer_connection->sk->sk_receive_queue)) { + wg_dbg("wg_tcp_listener_worker calling wg_tcp_data_ready() for temp peer\n"); + wg_tcp_data_ready(new_peer_connection->sk); + } + print_peer_socket_info(new_temp_peer); + wg_finish_tcp_connection_init(wg, + new_peer_connection); + mutex_unlock(&new_temp_peer->tcp_socket_lock); + } else { + kernel_sock_shutdown(new_peer_connection, SHUT_RDWR); + sock_release(new_peer_connection); + } + } + } + wg_dbg("Exiting function wg_tcp_listener_worker\n"); + return 0; +} + +int wg_tcp_listener4_thread(void *data) +{ + wg_dbg("Entering function wg_tcp_listener4_thread\n"); + struct wg_device *wg = data; + struct socket *listen_socket; + + /* Check if tcp_socket4_ready is set */ + if (!wg->tcp_socket4_ready) { + wg_dbg("tcp_socket4 is not ready, exiting wg_tcp_listener4_thread\n"); + return 0; + } + listen_socket = wg->tcp_listen_socket4; + + wg_dbg("Exiting function wg_tcp_listener4_thread\n"); + return wg_tcp_listener_worker(wg, listen_socket); +} + +int wg_tcp_listener6_thread(void *data) +{ + wg_dbg("Entering function wg_tcp_listener6_thread\n"); + struct wg_device *wg = data; + struct socket *listen_socket; + + if (!wg->tcp_socket6_ready) { + wg_dbg("tcp_socket6 is not ready, exiting wg_tcp_listener6_thread\n"); + return 0; + } + + listen_socket = wg->tcp_listen_socket6; + + wg_dbg("Exiting function wg_tcp_listener6_thread\n"); + return wg_tcp_listener_worker(wg, listen_socket); +} + +void wg_tcp_listener_socket_release(struct wg_device *wg) +{ + wg_dbg("Entering function wg_tcp_socket_release\n"); + + /* Wake blocking kernel_accept() calls before waiting for the listener + * threads. kthread_stop() alone does not make the accept wait condition + * true and can otherwise wait indefinitely. + */ + if (wg->tcp_listen_socket4) + kernel_sock_shutdown(wg->tcp_listen_socket4, SHUT_RDWR); +#if IS_ENABLED(CONFIG_IPV6) + if (wg->tcp_listen_socket6) + kernel_sock_shutdown(wg->tcp_listen_socket6, SHUT_RDWR); +#endif + + if (wg->tcp_listener4_thread) { + wg_dbg("Stopping IPv4 listener thread\n"); + kthread_stop(wg->tcp_listener4_thread); + wg->tcp_listener4_thread = NULL; + } + +#if IS_ENABLED(CONFIG_IPV6) + if (wg->tcp_listener6_thread) { + wg_dbg("Stopping IPv6 listener thread\n"); + kthread_stop(wg->tcp_listener6_thread); + wg->tcp_listener6_thread = NULL; + } +#endif + + /* Release IPv4 socket */ + if (wg->tcp_listen_socket4) { + wg_dbg("Releasing IPv4 socket\n"); + sock_release(wg->tcp_listen_socket4); + wg->tcp_listen_socket4 = NULL; + wg->tcp_socket4_ready = false; + } + +#if IS_ENABLED(CONFIG_IPV6) + if (wg->tcp_listen_socket6) { + wg_dbg("Releasing IPv6 socket\n"); + sock_release(wg->tcp_listen_socket6); + wg->tcp_listen_socket6 = NULL; + wg->tcp_socket6_ready = false; + } +#endif + wg->tcp_socket4_ready = false; + wg->tcp_socket6_ready = false; + + wg_dbg("Exiting function wg_tcp_socket_release\n"); +} + +int wg_setup_tcp_listen4(struct wg_device *wg, struct net *net, u16 port, + struct socket **listen_socket) +{ + struct socket *socket = NULL; + struct sockaddr_in addr4 = { + .sin_family = AF_INET, + .sin_port = htons(port), + .sin_addr = { htonl(INADDR_ANY) } + }; + int ret; + + if (!wg || !net || !listen_socket || port == 0) { + printk(KERN_ERR "wg_setup_tcp_listen4: Invalid arguments\n"); + return -EINVAL; + } + *listen_socket = NULL; + wg_dbg("Entering function wg_setup_tcp_listen4\n"); + + wg_dbg("Creating IPv4 socket\n"); + ret = sock_create_kern(net, AF_INET, SOCK_STREAM, IPPROTO_TCP, &socket); + if (ret < 0) { + pr_err("%s: Could not create IPv4 TCP socket, error: %d\n", wg->dev->name, ret); + return ret; + } + WRITE_ONCE(socket->sk->sk_mark, wg->fwmark); + wg_dbg("IPv4 socket created successfully\n"); + + /* Set socket options to reuse port */ + sock_set_reuseport(socket->sk); + + wg_dbg("Binding IPv4 socket\n"); + ret = kernel_bind(socket, (struct sockaddr *)&addr4, sizeof(addr4)); + if (ret < 0) { + pr_err("%s: Could not bind IPv4 TCP socket, error: %d\n", wg->dev->name, ret); + goto error; + } + wg_dbg("IPv4 socket bound successfully\n"); + + wg_dbg("Starting to listen on IPv4 socket\n"); + ret = kernel_listen(socket, SOMAXCONN); + if (ret < 0) { + pr_err("%s: Could not listen on IPv4 TCP socket, error: %d\n", wg->dev->name, ret); + goto error; + } + wg_dbg("IPv4 socket is now listening\n"); + *listen_socket = socket; + wg_dbg("Exiting function wg_setup_tcp_listen4 with ret=%d\n", ret); + return 0; + +error: + sock_release(socket); + wg_dbg("Exiting function wg_setup_tcp_listen4 with ret=%d\n", ret); + return ret; +} + +int wg_setup_tcp_listen6(struct wg_device *wg, struct net *net, u16 port, + struct socket **listen_socket) +{ +#if IS_ENABLED(CONFIG_IPV6) + struct socket *socket = NULL; + struct sockaddr_in6 addr6 = { + .sin6_family = AF_INET6, + .sin6_port = htons(port), + .sin6_addr = IN6ADDR_ANY_INIT, + }; + int ret; + + if (!wg || !net || !listen_socket || port == 0) { + printk(KERN_ERR "wg_setup_tcp_listen6: Invalid arguments\n"); + return -EINVAL; + } + *listen_socket = NULL; + wg_dbg("Entering function wg_setup_tcp_listen6\n"); + + wg_dbg("Creating IPv6 socket\n"); + ret = sock_create_kern(net, AF_INET6, SOCK_STREAM, IPPROTO_TCP, &socket); + if (ret < 0) { + pr_err("%s: Could not create IPv6 TCP socket, error: %d\n", wg->dev->name, ret); + return ret; + } + WRITE_ONCE(socket->sk->sk_mark, wg->fwmark); + wg_dbg("IPv6 socket created successfully\n"); + + /* Keep the IPv4 and IPv6 wildcard listeners independent. */ + ret = ip6_sock_set_v6only(socket->sk); + if (ret < 0) { + pr_err("%s: Could not make IPv6 TCP listener v6-only, error: %d\n", + wg->dev->name, ret); + goto error; + } + + wg_dbg("Binding IPv6 socket\n"); + ret = kernel_bind(socket, (struct sockaddr *)&addr6, sizeof(addr6)); + if (ret < 0) { + pr_err("%s: Could not bind IPv6 TCP socket, error: %d\n", wg->dev->name, ret); + goto error; + } + wg_dbg("IPv6 socket bound successfully\n"); + + wg_dbg("Starting to listen on IPv6 socket\n"); + ret = kernel_listen(socket, SOMAXCONN); + if (ret < 0) { + pr_err("%s: Could not listen on IPv6 TCP socket, error: %d\n", wg->dev->name, ret); + goto error; + } + wg_dbg("IPv6 socket is now listening\n"); + *listen_socket = socket; + wg_dbg("Exiting function wg_setup_tcp_listen6 with ret=%d\n", ret); + return 0; + +error: + sock_release(socket); + wg_dbg("Exiting function wg_setup_tcp_listen6 with ret=%d\n", ret); + return ret; +#else + return -EAFNOSUPPORT; +#endif +} + +int wg_tcp_listener_socket_init(struct wg_device *wg, u16 port) +{ + struct socket *listen_socket4 = NULL, *listen_socket6 = NULL; + struct net *net; + int ret; + + if (!wg || port == 0) { + printk(KERN_ERR "wg_tcp_listener_socket_init: Invalid arguments\n"); + return -EINVAL; + } + wg_dbg("Entering function wg_tcp_listener_socket_init\n"); + + if (wg->tcp_socket4_ready || wg->tcp_socket6_ready) { + wg_dbg("TCP sockets are already initialized, exiting\n"); + return 0; + } + + if (!wg->dev) { + wg_dbg("Net Device not initialized in wg_device, exiting\n"); + return -EINVAL; + } + + wg_dbg("Locking RCU and dereferencing wg->creating_net\n"); + rcu_read_lock(); + net = rcu_dereference(wg->creating_net); + net = net ? maybe_get_net(net) : NULL; + rcu_read_unlock(); + wg_dbg("RCU lock released\n"); + + if (unlikely(!net)) { + printk(KERN_ERR "Error: net is NULL, exiting wg_tcp_listener_socket_init\n"); + return -ENONET; + } + + + + + /* Match the UDP transport's family policy: IPv4 is required and IPv6 is + * added when the module is available. Wildcard binds do not require a + * default route or a globally selected interface. + */ + ret = wg_setup_tcp_listen4(wg, net, port, &listen_socket4); + if (ret < 0) + goto error_sockets; + +#if IS_ENABLED(CONFIG_IPV6) + if (ipv6_mod_enabled()) { + ret = wg_setup_tcp_listen6(wg, net, port, &listen_socket6); + if (ret < 0) + goto error_sockets; + } +#endif + + if (!listen_socket4 && !listen_socket6) { + ret = -EADDRNOTAVAIL; + pr_err("%s: No address family is available for a TCP listener\n", + wg->dev->name); + goto error_sockets; + } + + wg->tcp_listen_socket4 = listen_socket4; + wg->tcp_listen_socket6 = listen_socket6; + wg->tcp_socket4_ready = listen_socket4 != NULL; + wg->tcp_socket6_ready = listen_socket6 != NULL; + + if (wg->tcp_listen_socket4) { + wg_dbg("Starting IPv4 listener thread\n"); + wg->tcp_listener4_thread = kthread_run(wg_tcp_listener4_thread, + (void *)wg, "wg_listener4"); + if (IS_ERR(wg->tcp_listener4_thread)) { + ret = PTR_ERR(wg->tcp_listener4_thread); + wg->tcp_listener4_thread = NULL; + pr_err("%s: Failed to establish IPv4 TCP listener thread: %d\n", + wg->dev->name, ret); + goto error_listeners; + } + wg_dbg("IPv4 listener thread started successfully\n"); + } + +#if IS_ENABLED(CONFIG_IPV6) + if (wg->tcp_listen_socket6) { + wg_dbg("Starting IPv6 listener thread\n"); + wg->tcp_listener6_thread = kthread_run(wg_tcp_listener6_thread, + (void *)wg, "wg_listener6"); + if (IS_ERR(wg->tcp_listener6_thread)) { + ret = PTR_ERR(wg->tcp_listener6_thread); + wg->tcp_listener6_thread = NULL; + pr_err("%s: Failed to establish IPv6 TCP listener thread: %d\n", + wg->dev->name, ret); + goto error_listeners; + } + wg_dbg("IPv6 listener thread started successfully\n"); + } +#endif + + put_net(net); + wg_dbg("Exiting function wg_tcp_listener_socket_init\n"); + return 0; + +error_listeners: + wg_tcp_listener_socket_release(wg); + goto out_net; +error_sockets: + if (listen_socket4) + sock_release(listen_socket4); +#if IS_ENABLED(CONFIG_IPV6) + if (listen_socket6) + sock_release(listen_socket6); +#endif +out_net: + put_net(net); + wg_dbg("Exiting function wg_tcp_listener_socket_init with error: %d\n", ret); + return ret; +} +static void wg_tcp_connect_unwind(struct wg_peer *peer, struct socket *socket) +{ + bool queue_reconnect = false; + bool owns_socket = false; + int detach_ret = 0; + + lockdep_assert_held(&peer->tcp_socket_lock); + + /* A connect callback can publish ESTABLISHED before kernel_connect() + * returns. Claim removal and drain any writer queued in that window before + * releasing a failed connection attempt. + */ + spin_lock_bh(&peer->tcp_lock); + if (socket && (peer->peer_socket == socket || + peer->outbound_socket == socket)) { + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = socket; + owns_socket = true; + } + spin_unlock_bh(&peer->tcp_lock); + if (owns_socket) { + cancel_work_sync(&peer->tcp_read_work); + cancel_work_sync(&peer->tcp_write_work); + spin_lock_bh(&peer->tcp_lock); + spin_lock(&peer->tcp_read_lock); + peer->tcp_read_worker_scheduled = false; + spin_unlock(&peer->tcp_read_lock); + spin_lock(&peer->tcp_write_lock); + peer->tcp_write_worker_scheduled = false; + spin_unlock(&peer->tcp_write_lock); + spin_unlock_bh(&peer->tcp_lock); + } + + /* Stop WireGuard callbacks and detach their wrapper while the socket is + * still alive. This waits for any callback already holding callback_lock. + */ + if (owns_socket) { + detach_ret = wg_reset_exact_tcp_socket_callbacks(peer, socket); + if (!detach_ret) + detach_ret = wg_release_peer_socket_locked(peer, socket); + } + if (detach_ret) { + spin_lock_bh(&peer->tcp_lock); + peer->tcp_connecting = false; + spin_unlock_bh(&peer->tcp_lock); + WARN_ON_ONCE(detach_ret); + return; + } + + /* Publish one coherent disconnected state before releasing the socket. + * Consumers either see this state or the still-live socket above. + */ + spin_lock_bh(&peer->tcp_lock); + peer->tcp_connecting = false; + peer->tcp_pending = false; + peer->tcp_established = false; + peer->outbound_connected = false; + peer->tcp_outbound_remove_socket = NULL; + if (peer->tcp_reconnect_requested && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + peer->device->transport == WG_TRANSPORT_TCP) { + peer->tcp_outbound_remove_scheduled = true; + queue_reconnect = true; + } else if (!peer->tcp_stopping) { + peer->tcp_outbound_remove_scheduled = false; + } + peer->clean_outbound = false; + peer->outbound_timestamp = ktime_set(0, 0); + spin_unlock_bh(&peer->tcp_lock); + + if (socket && !owns_socket) + sock_release(socket); + if (queue_reconnect) { + /* The socket is gone before replacement work can run. Recheck the + * stop barrier under the ownership lock so peer_stop cannot drain the + * work item and then lose a late queue. + */ + spin_lock_bh(&peer->tcp_lock); + if (!peer->tcp_stopping && peer->tcp_reconnect_requested && + peer->tcp_outbound_remove_scheduled) + mod_delayed_work(system_wq, + &peer->tcp_outbound_remove_work, 0); + spin_unlock_bh(&peer->tcp_lock); + } +} + +/* Publish the first established observation for one exact outbound carrier. + * Both kernel_connect() and the state callback can observe that transition, so + * tcp_lock elects exactly one authenticated bootstrap sender. + */ +static bool +wg_tcp_publish_outbound_established_locked(struct wg_peer *peer, + struct socket *socket) +{ + bool first_observation; + + lockdep_assert_held(&peer->tcp_lock); + if (!socket || peer->peer_socket != socket || + peer->outbound_socket != socket) + return false; + first_observation = !peer->tcp_established || + !peer->outbound_connected; + peer->tcp_pending = false; + peer->tcp_established = true; + peer->outbound_connected = true; + if (first_observation) + peer->outbound_timestamp = ktime_get(); + return first_observation; +} + +static void wg_tcp_send_carrier_bootstrap(struct wg_peer *peer, + struct socket *socket) +{ + bool is_current; + + /* TCP establishment is not peer authentication. Sending a keepalive + * emits an authenticated record when a key exists, or starts a handshake + * otherwise, allowing the listener to promote its provisional carrier. + */ + spin_lock_bh(&peer->tcp_lock); + is_current = !READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + peer->device->transport == WG_TRANSPORT_TCP && + netif_running(peer->device->dev) && + !peer->tcp_outbound_remove_scheduled && + peer->peer_socket == socket && + peer->outbound_socket == socket && peer->tcp_established && + peer->outbound_connected; + spin_unlock_bh(&peer->tcp_lock); + if (is_current) + wg_packet_send_keepalive(peer); +} + +static void wg_tcp_queue_carrier_bootstrap(struct wg_peer *peer, + struct socket *socket) +{ + spin_lock_bh(&peer->tcp_lock); + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + peer->peer_socket == socket && peer->outbound_socket == socket) { + peer->tcp_bootstrap_socket = socket; + queue_work(system_wq, &peer->tcp_bootstrap_work); + } + spin_unlock_bh(&peer->tcp_lock); +} + +void wg_tcp_bootstrap_worker(struct work_struct *work) +{ + struct wg_peer *peer = + container_of(work, struct wg_peer, tcp_bootstrap_work); + struct socket *socket; + + /* State-change callbacks run with sk_callback_lock_bh held and may not + * enter the Noise send path directly. Move that work to process context + * and serialize the exact carrier with socket teardown. + */ + mutex_lock(&peer->tcp_socket_lock); + spin_lock_bh(&peer->tcp_lock); + socket = peer->tcp_bootstrap_socket; + peer->tcp_bootstrap_socket = NULL; + spin_unlock_bh(&peer->tcp_lock); + if (socket) + wg_tcp_send_carrier_bootstrap(peer, socket); + mutex_unlock(&peer->tcp_socket_lock); +} + +/* Attempt to establish a TCP connection */ +int wg_tcp_connect(struct wg_peer *peer) +{ + struct socket *socket = NULL; + struct net *net; + struct endpoint target; + struct sockaddr_storage addr_storage; + struct sockaddr *addr = (struct sockaddr *)&addr_storage; + unsigned long timeout = 30 * HZ; + struct socket *bootstrap_socket = NULL; + bool queue_remove = false; + bool queue_retry = false; + int family; + int ret; + + if (!peer || IS_ERR(peer) || !peer->device) + return -EINVAL; + + wg_dbg("Entering function wg_tcp_connect peer=%px\n", peer); + print_peer_socket_info(peer); + + if (peer->device->transport != WG_TRANSPORT_TCP) { + pr_err("Invalid state for TCP connection attempt.\n"); + return -EINVAL; + } + mutex_lock(&peer->tcp_socket_lock); + + /* One connect attempt must use one coherent target even if an + * authenticated packet or netlink update changes the next retry target. + */ + read_lock_bh(&peer->endpoint_lock); + if (peer->peer_endpoint_set) + target = peer->peer_endpoint; + else + memset(&target, 0, sizeof(target)); + read_unlock_bh(&peer->endpoint_lock); + family = target.addr.sa_family; + + wg_dbg("(Device) Peer transport: %d, TCP established: %d\n", + peer->device->transport, peer->tcp_established); + wg_dbg("Peer endpoint address family: %d\n", family); + log_wireguard_endpoint(&target); + + if (family != AF_INET && family != AF_INET6) { + printk(KERN_ERR "Invalid address family for connection: %d\n", + family); + ret = -EAFNOSUPPORT; + goto out_unlock; + } + + /* tcp_pending is also the connect-attempt ownership claim. It prevents + * retry, send, and endpoint-update paths from publishing a second socket. + */ + spin_lock_bh(&peer->tcp_lock); + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP || + !netif_running(peer->device->dev) || peer->tcp_established || + peer->tcp_pending || + peer->inbound_connected || peer->outbound_connected || + peer->tcp_outbound_remove_scheduled) { + spin_unlock_bh(&peer->tcp_lock); + ret = 0; + goto out_unlock; + } + if (peer->peer_socket || peer->outbound_socket) { + spin_unlock_bh(&peer->tcp_lock); + ret = -EALREADY; + goto out_unlock; + } + peer->tcp_connecting = true; + peer->tcp_pending = true; + peer->tcp_established = false; + peer->outbound_connected = false; + peer->tcp_outbound_callbacks_set = false; + peer->outbound_timestamp = ktime_set(0, 0); + spin_unlock_bh(&peer->tcp_lock); + + memset(&addr_storage, 0, sizeof(addr_storage)); + + if (family == AF_INET) { + struct sockaddr_in *addr4 = (struct sockaddr_in *)&addr_storage; + addr4->sin_family = AF_INET; + addr4->sin_port = target.addr4.sin_port; + addr4->sin_addr.s_addr = target.addr4.sin_addr.s_addr; + addr = (struct sockaddr *)addr4; + wg_dbg("Setting up IPv4 connection to %pI4:%d\n", &addr4->sin_addr, ntohs(addr4->sin_port)); + } +#ifdef CONFIG_IPV6 + else if (family == AF_INET6) { + struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&addr_storage; + addr6->sin6_family = AF_INET6; + addr6->sin6_port = target.addr6.sin6_port; + addr6->sin6_addr = target.addr6.sin6_addr; + addr6->sin6_scope_id = target.addr6.sin6_scope_id; + addr = (struct sockaddr *)addr6; + wg_dbg("Setting up IPv6 connection to [%pI6c]:%d\n", &addr6->sin6_addr, ntohs(addr6->sin6_port)); + } +#endif + else { + pr_err("Unsupported address family: %d\n", family); + wg_dbg("Exiting function wg_tcp_connect\n"); + ret = -EAFNOSUPPORT; + goto fail; + } + + /* The device can outlive a move into another namespace, so use the + * retained creation namespace just as the UDP and TCP listeners do. + */ + rcu_read_lock(); + net = rcu_dereference(peer->device->creating_net); + net = net ? maybe_get_net(net) : NULL; + rcu_read_unlock(); + if (unlikely(!net)) { + ret = -ENONET; + goto fail; + } + + /* Create the socket */ + wg_dbg("Creating socket for address family: %d\n", family); + ret = sock_create_kern(net, family, + SOCK_STREAM, IPPROTO_TCP, &socket); + put_net(net); + if (ret) { + pr_err("Failed to create TCP socket for address family %d: %d\n", + family, ret); + wg_dbg("Exiting function wg_tcp_connect\n"); + goto fail; + } + WRITE_ONCE(socket->sk->sk_mark, peer->device->fwmark); + spin_lock_bh(&peer->tcp_lock); + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP || + !netif_running(peer->device->dev) || !peer->tcp_connecting || + !peer->tcp_pending || peer->peer_socket || peer->outbound_socket) { + spin_unlock_bh(&peer->tcp_lock); + ret = -ESHUTDOWN; + goto fail; + } + peer->peer_socket = socket; + peer->outbound_socket = socket; + spin_unlock_bh(&peer->tcp_lock); + + /* Print diagnostic information about the created socket */ + wg_dbg("Socket created, sk=%px, family=%d, state=%d\n", + socket->sk, socket->sk->sk_family, socket->sk->sk_state); + + /* Set up the socket callbacks before initiating the connect */ + wg_dbg("Setting up socket callbacks\n"); + ret = wg_setup_tcp_socket_callbacks(peer, socket, false); + if (ret) + goto fail; + + /* Set socket timeouts for send and receive operations */ + wg_dbg("Setting socket timeouts\n"); + ret = wg_set_socket_timeouts(socket, timeout, timeout); + if (ret) { + pr_err("Failed to set socket timeouts: %d\n", ret); + goto fail; + } + + wg_dbg("Ready to initiate connection, sk_state=%d\n", + socket->sk->sk_state); + + /* Initiate the non-blocking connect */ + wg_dbg("Initiating non-blocking connect\n"); + ret = kernel_connect(socket, addr, + addr->sa_family == AF_INET ? + sizeof(struct sockaddr_in) : + sizeof(struct sockaddr_in6), + O_NONBLOCK); + + /* FIX #4: Disable Nagle's algorithm on outbound socket to avoid + * ~200ms delayed ACK interaction that caused 1000ms RTT + */ + tcp_sock_set_nodelay(socket->sk); + + if (ret != -EINPROGRESS && ret != 0) { + pr_err("TCP connection attempt failed: %d\n", ret); + goto fail; + } + + /* kernel_connect() selects the route, local address, and ephemeral source + * port. Cache the tuple only after that selection; the pre-connect socket + * fields are commonly still zero. + */ + { + struct inet_sock *inet = inet_sk(socket->sk); + + memset(&peer->outbound_source, 0, + sizeof(peer->outbound_source)); + memset(&peer->outbound_dest, 0, sizeof(peer->outbound_dest)); + if (family == AF_INET) { + struct sockaddr_in *source = + (struct sockaddr_in *)&peer->outbound_source; + struct sockaddr_in *dest = + (struct sockaddr_in *)&peer->outbound_dest; + + source->sin_family = AF_INET; + source->sin_port = inet->inet_sport; + source->sin_addr.s_addr = inet->inet_saddr; + dest->sin_family = AF_INET; + dest->sin_port = inet->inet_dport; + dest->sin_addr.s_addr = inet->inet_daddr; +#ifdef CONFIG_IPV6 + } else if (family == AF_INET6) { + struct sockaddr_in6 *source6 = + (struct sockaddr_in6 *)&peer->outbound_source; + struct sockaddr_in6 *dest6 = + (struct sockaddr_in6 *)&peer->outbound_dest; + + source6->sin6_family = AF_INET6; + source6->sin6_port = inet->inet_sport; + source6->sin6_addr = inet6_sk(socket->sk)->saddr; + dest6->sin6_family = AF_INET6; + dest6->sin6_port = inet->inet_dport; + dest6->sin6_addr = socket->sk->sk_v6_daddr; + dest6->sin6_scope_id = target.addr6.sin6_scope_id; +#endif + } + } + + wg_dbg("TCP connection attempt initiated\n"); + spin_lock_bh(&peer->tcp_lock); + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP || + !netif_running(peer->device->dev) || + peer->peer_socket != socket || peer->outbound_socket != socket || + (READ_ONCE(socket->sk->sk_state) != TCP_SYN_SENT && + READ_ONCE(socket->sk->sk_state) != TCP_SYN_RECV && + READ_ONCE(socket->sk->sk_state) != TCP_ESTABLISHED)) { + spin_unlock_bh(&peer->tcp_lock); + ret = -ECONNABORTED; + goto fail; + } + peer->tcp_connecting = false; + if (READ_ONCE(socket->sk->sk_state) == TCP_ESTABLISHED && + wg_tcp_publish_outbound_established_locked(peer, socket)) + bootstrap_socket = socket; + if (peer->tcp_reconnect_requested && !peer->tcp_stopping && + !peer->tcp_outbound_remove_scheduled) { + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = socket; + queue_remove = true; + } else if (peer->tcp_pending && !peer->tcp_retry_scheduled) { + peer->tcp_retry_scheduled = true; + queue_retry = true; + } + if (queue_remove) + mod_delayed_work(system_wq, &peer->tcp_outbound_remove_work, 0); + if (queue_retry) { + wg_dbg("Scheduling TCP retry work.\n"); + mod_delayed_work(system_wq, &peer->tcp_retry_work, + msecs_to_jiffies(10000)); + } + spin_unlock_bh(&peer->tcp_lock); + + wg_dbg("Exiting function wg_tcp_connect\n"); + mutex_unlock(&peer->tcp_socket_lock); + if (bootstrap_socket) + wg_tcp_queue_carrier_bootstrap(peer, bootstrap_socket); + return 0; + +fail: + wg_tcp_connect_unwind(peer, socket); + wg_dbg("Exiting function wg_tcp_connect with error: %d\n", ret); +out_unlock: + mutex_unlock(&peer->tcp_socket_lock); + return ret; +} + +/* Function to release and clean up an old peer TCP connection - clean the active connection */ +static void __maybe_unused wg_release_peer_tcp_connection(struct wg_peer *peer) +{ + struct socket *socket; + int ret; + + if (!peer || IS_ERR(peer)) + return; + mutex_lock(&peer->tcp_socket_lock); + spin_lock_bh(&peer->tcp_lock); + socket = peer->peer_socket; + spin_unlock_bh(&peer->tcp_lock); + if (socket) { + ret = wg_reset_exact_tcp_socket_callbacks(peer, socket); + if (!ret) + ret = wg_release_peer_socket_locked(peer, socket); + WARN_ON_ONCE(ret); + } + mutex_unlock(&peer->tcp_socket_lock); +} + + +void wg_extract_endpoint_from_sock(struct sock *sk, + struct endpoint *endpoint) +{ + wg_dbg("Entering function wg_extract_endpoint_from_sock\n"); + if (!sk || !endpoint) { + pr_warn("Socket or endpoint is NULL.\n"); + return; + } + memset(endpoint, 0, sizeof(*endpoint)); /* Clear the endpoint structure */ + + if (sk->sk_family == AF_INET) { + /* IPv4 */ + struct inet_sock *inet = inet_sk(sk); + + endpoint->addr4.sin_family = AF_INET; + endpoint->addr4.sin_port = inet->inet_dport; /* Destination port */ + endpoint->addr4.sin_addr.s_addr = inet->inet_daddr; /* Destination IP address */ + } else if (sk->sk_family == AF_INET6) { +#if IS_ENABLED(CONFIG_IPV6) + /* IPv6 */ + endpoint->addr6.sin6_family = AF_INET6; + endpoint->addr6.sin6_port = sk->sk_dport; /* Destination port */ + endpoint->addr6.sin6_addr = sk->sk_v6_daddr; /* Destination IP address */ + + if (ipv6_addr_type((struct in6_addr *)&sk->sk_v6_daddr) & IPV6_ADDR_LINKLOCAL) { + /* + * Preserve the bound interface as the scope of a + * link-local destination. + */ + endpoint->addr6.sin6_scope_id = sk->sk_bound_dev_if; + } else { + /* Not a link-local address; no scope ID required */ + endpoint->addr6.sin6_scope_id = 0; + } + } else { +#endif + pr_warn("Unsupported socket family: %d.\n", sk->sk_family); + } + wg_dbg("Exiting function wg_extract_endpoint_from_sock\n"); +} + + +void wg_tcp_state_change(struct sock *sk) +{ + struct wg_device *cleanup_device = NULL; + struct wg_socket_data *socket_data = NULL; + struct wg_peer *peer = NULL; + struct socket *bootstrap_socket = NULL; + void (*original_state_change)(struct sock *) = NULL; + bool cleanup_temp = false; + bool cancel_retry = false; + bool queue_inbound_remove = false; + bool queue_outbound_remove = false; + + wg_dbg("Entering function wg_tcp_state_change\n"); + + /* Check if the socket is valid */ + if (!sk || IS_ERR(sk)) { + pr_err("wg_tcp_state_change: Invalid socket passed to the function\n"); + goto done; + } + + read_lock_bh(&sk->sk_callback_lock); + + /* Retrieve the socket user data */ + socket_data = sk->sk_user_data; + + /* Check if socket_data is valid */ + if (!socket_data || IS_ERR(socket_data)) { + pr_err("wg_tcp_state_change: Invalid or NULL socket_data for socket %px\n", sk); + goto unlock; + } + + /* Retrieve the peer from the socket_data */ + peer = socket_data->peer; + + /* Check if peer is valid or being torn down */ + if (!peer || IS_ERR(peer)) + goto unlock; + original_state_change = socket_data->original_state_change; + if (READ_ONCE(peer->is_dead) || + (!socket_data->inbound && + READ_ONCE(peer->tcp_outbound_remove_scheduled)) || + (socket_data->inbound && + READ_ONCE(peer->tcp_inbound_remove_scheduled))) { + goto unlock; + } + print_peer_socket_info(peer); + /* Diagnostic information about the current state */ +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "state_change", 0, 0); +#endif + wg_dbg("wg_tcp_state_change: Socket state=%d, Socket error=%d\n", sk->sk_state, sk->sk_err); + wg_dbg("wg_tcp_state_change: Peer=%px, Device=%px\n", peer, socket_data->device); + + /* Additional diagnostic information for peer-specific data */ + wg_dbg("wg_tcp_state_change: Peer TCP established=%d, TCP pending=%d\n", + peer->tcp_established, peer->tcp_pending); + + + + /* Log detailed state information */ + wg_dbg("wg_tcp_state_change: sk=%px, sk_state=%d, sk_err=%d, sk_shutdown=%d, sk_send_head=%px\n", + sk, sk->sk_state, sk->sk_err, sk->sk_shutdown, sk->sk_send_head); + /* Log TCP specific state information if available */ + const char *tcp_state_name; + + switch (sk->sk_state) { + case TCP_ESTABLISHED: + tcp_state_name = "TCP_ESTABLISHED"; + break; + case TCP_SYN_SENT: + tcp_state_name = "TCP_SYN_SENT"; + break; + case TCP_SYN_RECV: + tcp_state_name = "TCP_SYN_RECV"; + break; + case TCP_FIN_WAIT1: + tcp_state_name = "TCP_FIN_WAIT1"; + break; + case TCP_FIN_WAIT2: + tcp_state_name = "TCP_FIN_WAIT2"; + break; + case TCP_TIME_WAIT: + tcp_state_name = "TCP_TIME_WAIT"; + break; + case TCP_CLOSE: + tcp_state_name = "TCP_CLOSE"; + break; + case TCP_CLOSE_WAIT: + tcp_state_name = "TCP_CLOSE_WAIT"; + break; + case TCP_LAST_ACK: + tcp_state_name = "TCP_LAST_ACK"; + break; + case TCP_LISTEN: + tcp_state_name = "TCP_LISTEN"; + break; + case TCP_CLOSING: + tcp_state_name = "TCP_CLOSING"; + break; + case TCP_NEW_SYN_RECV: + tcp_state_name = "TCP_NEW_SYN_RECV"; + break; + default: + tcp_state_name = "UNKNOWN_STATE"; + break; + } + + wg_dbg("TCP state: %s (%d)\n", tcp_state_name, sk->sk_state); + + if (sk->sk_state == TCP_ESTABLISHED) { + struct tcp_sock *tp = tcp_sk(sk); + wg_dbg("TCP_ESTABLISHED: snd_una=%u, snd_nxt=%u, snd_wnd=%u, rcv_wnd=%u, rcv_nxt=%u\n", + tp->snd_una, tp->snd_nxt, tp->snd_wnd, tp->rcv_wnd, tp->rcv_nxt); + } + + /* first lets figure out if this is an inbound connect */ + + switch (sk->sk_state) { + case TCP_ESTABLISHED: + if (peer->temp_peer) { + pr_err("Wireguard: Inbound peer connection previously established.\n"); + break; + } + if (socket_data->inbound) + break; + spin_lock_bh(&peer->tcp_lock); + if (peer->outbound_socket && + peer->outbound_socket->sk == sk) { + if (wg_tcp_publish_outbound_established_locked( + peer, peer->outbound_socket)) + bootstrap_socket = peer->outbound_socket; + if (peer->tcp_retry_scheduled) { + peer->tcp_retry_scheduled = false; + cancel_retry = true; + } + wg_dbg("TCP connection established.\n"); + } else + pr_err("Wireguard: Outbound connection previously established.\n"); + spin_unlock_bh(&peer->tcp_lock); + if (cancel_retry) + cancel_delayed_work(&peer->tcp_retry_work); + break; + case TCP_CLOSE: + case TCP_CLOSE_WAIT: + case TCP_CLOSING: + case TCP_FIN_WAIT1: + case TCP_FIN_WAIT2: + case TCP_LAST_ACK: + if (peer->temp_peer) { + WRITE_ONCE(peer->is_dead, true); + cleanup_device = peer->device; + cleanup_temp = true; + break; + } + wg_dbg("TCP connection failed or closed, handling state.\n"); + spin_lock_bh(&peer->tcp_lock); + if (socket_data->inbound) { + if (!peer->inbound_socket || + peer->inbound_socket->sk != sk) { + spin_unlock_bh(&peer->tcp_lock); + break; + } + peer->inbound_timestamp = ktime_set(0, 0); + peer->inbound_connected = false; + if (!peer->tcp_inbound_remove_scheduled) { + peer->tcp_inbound_remove_scheduled = true; + peer->tcp_inbound_remove_socket = + peer->inbound_socket; + queue_inbound_remove = true; + } + } else { + if (!peer->outbound_socket || + peer->outbound_socket->sk != sk) { + spin_unlock_bh(&peer->tcp_lock); + break; + } + peer->outbound_timestamp = ktime_set(0, 0); + peer->outbound_connected = false; + peer->tcp_pending = false; + if (peer->tcp_connecting) { + spin_unlock_bh(&peer->tcp_lock); + break; + } + peer->tcp_reconnect_requested = true; + if (!peer->tcp_outbound_remove_scheduled) { + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = + peer->outbound_socket; + queue_outbound_remove = true; + } + } + if (!peer->inbound_connected && !peer->outbound_connected) + peer->tcp_established = false; + spin_unlock_bh(&peer->tcp_lock); + break; + default: + break; + } +unlock: + if (original_state_change) + original_state_change(sk); + if (bootstrap_socket) + wg_tcp_queue_carrier_bootstrap(peer, bootstrap_socket); + if (peer && (queue_inbound_remove || queue_outbound_remove)) { + /* The original callback can overlap device or peer stop. Recheck the + * barrier and publish work while holding tcp_lock so a completed + * cancel_delayed_work_sync() cannot be followed by a late queue. + */ + spin_lock_bh(&peer->tcp_lock); + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled)) { + if (queue_inbound_remove && + peer->tcp_inbound_remove_scheduled && + peer->tcp_inbound_remove_socket == + peer->inbound_socket && + peer->inbound_socket && + peer->inbound_socket->sk == sk) + mod_delayed_work(system_wq, + &peer->tcp_inbound_remove_work, 0); + if (queue_outbound_remove && + peer->tcp_outbound_remove_scheduled && + peer->tcp_outbound_remove_socket == + peer->outbound_socket && + peer->outbound_socket && + peer->outbound_socket->sk == sk) + mod_delayed_work(system_wq, + &peer->tcp_outbound_remove_work, 0); + } + spin_unlock_bh(&peer->tcp_lock); + } + if (cleanup_temp && READ_ONCE(cleanup_device->tcp_cleanup_scheduled)) + mod_delayed_work(system_wq, &cleanup_device->tcp_cleanup_work, 0); + read_unlock_bh(&sk->sk_callback_lock); +done: + wg_dbg("Exiting function wg_tcp_state_change\n"); +} + + + +void log_wireguard_endpoint(struct endpoint *ep) +{ +#ifndef WG_TCP_VERBOSE + return; +#else + char addr_str[INET6_ADDRSTRLEN]; + + if (!ep) { + wg_dbg("WireGuard: Endpoint is NULL.\n"); + return; + } + + switch (ep->addr.sa_family) { + case AF_INET: { + /* Handle IPv4 address */ + struct sockaddr_in *sin = &ep->addr4; + snprintf(addr_str, sizeof(addr_str), "%pI4", &sin->sin_addr); + wg_dbg("Endpoint IPv4: %s:%u\n", + addr_str, ntohs(sin->sin_port)); + if (ep->src_if4 != 0) { + snprintf(addr_str, sizeof(addr_str), "%pI4", &ep->src4); + wg_dbg("Source IPv4: %s, Source Interface: %d\n", + addr_str, ep->src_if4); + } + break; + } + case AF_INET6: { + /* Handle IPv6 address */ + struct sockaddr_in6 *sin6 = &ep->addr6; + snprintf(addr_str, sizeof(addr_str), "%pI6", &sin6->sin6_addr); + wg_dbg("Endpoint IPv6: [%s]:%u, Scope ID: %u\n", + addr_str, ntohs(sin6->sin6_port), sin6->sin6_scope_id); + snprintf(addr_str, sizeof(addr_str), "%pI6", &ep->src6); + wg_dbg("Source IPv6: [%s]\n", addr_str); + break; + } + default: + wg_dbg("Unsupported address family: %d\n", ep->addr.sa_family); + break; + } +#endif +} + + + +void wg_get_endpoint_from_socket(struct socket *epsocket, struct endpoint *ep) +{ + /* Validate input parameters */ + if (!epsocket || !ep) { + printk(KERN_ERR "Invalid input: epsocket or ep is NULL\n"); + return; + } + + /* Validate the socket's `sock` structure */ + if (!epsocket->sk) { + printk(KERN_ERR "Invalid socket: epsocket->sk is NULL\n"); + return; + } + + struct sock *sk = epsocket->sk; + int family = sk->sk_family; + + if (family == AF_INET) { + struct inet_sock *inet = inet_sk(sk); + + /* Validate inet_sk */ + if (!inet) { + printk(KERN_ERR "inet_sk is NULL for IPv4 socket\n"); + return; + } + + /* Ensure that the inet_daddr and inet_dport are valid before accessing */ + if (inet->inet_daddr == 0 || inet->inet_dport == 0) { + printk(KERN_ERR "Invalid IPv4 address or port\n"); + return; + } + + /* Populate the endpoint with IPv4 address and port */ + ep->addr4.sin_family = AF_INET; + ep->addr4.sin_addr.s_addr = inet->inet_daddr; /* Remote IPv4 address */ + ep->addr4.sin_port = inet->inet_dport; /* Remote port */ + + /* Populate src4 fields with local information */ + ep->src4.s_addr = inet->inet_saddr; /* Local IPv4 address */ + ep->src_if4 = sk->sk_bound_dev_if; /* Interface index */ + + /* Diagnostics */ + wg_dbg("IPv4 endpoint: remote %pI4:%u, local %pI4:%u\n", + &ep->addr4.sin_addr.s_addr, ntohs(ep->addr4.sin_port), + &ep->src4.s_addr, ntohs(inet->inet_sport)); + + } +#if IS_ENABLED(CONFIG_IPV6) + else if (family == AF_INET6) { + struct ipv6_pinfo *np = inet6_sk(sk); + + /* Validate ipv6_pinfo */ + if (!np) { + printk(KERN_ERR "ipv6_pinfo is NULL for IPv6 socket\n"); + return; + } + + /* Ensure that the IPv6 address and port are valid before accessing */ + if (ipv6_addr_any(&sk->sk_v6_daddr) || inet_sk(sk)->inet_dport == 0) { + printk(KERN_ERR "Invalid IPv6 address or port\n"); + return; + } + + /* Populate the endpoint with IPv6 address and port */ + ep->addr6.sin6_family = AF_INET6; + ep->addr6.sin6_addr = sk->sk_v6_daddr; /* Remote IPv6 address */ + ep->addr6.sin6_port = inet_sk(sk)->inet_dport; /* Remote port */ + ep->addr6.sin6_scope_id = ipv6_iface_scope_id(&sk->sk_v6_rcv_saddr, sk->sk_bound_dev_if); + + /* Populate src6 fields with local information */ + ep->src6 = sk->sk_v6_rcv_saddr; /* Local IPv6 address */ + + /* Diagnostics */ + wg_dbg("IPv6 endpoint: remote %pI6c:%u, local %pI6c:%u\n", + &ep->addr6.sin6_addr, ntohs(ep->addr6.sin6_port), + &ep->src6, ntohs(inet_sk(sk)->inet_sport)); + } +#endif + else { + printk(KERN_ERR "Unsupported address family: %d\n", family); + return; + } +} + +int wg_tcp_queuepkt(struct wg_peer *peer, const void *data, + size_t len) +{ + struct sk_buff *frame; + struct sk_buff *skb; + int ret; + + wg_dbg("Entering function wg_tcp_queuepkt peer=%px\n", peer); + + struct endpoint current_endpoint; + /* BUG FIX: current_endpoint was never initialized — reads garbage in log_wireguard_endpoint */ + memset(¤t_endpoint, 0, sizeof(current_endpoint)); + + if (!peer || IS_ERR(peer)) { + wg_dbg("Exiting function wg_tcp_queuepkt, no peer.\n"); + return -EINVAL; + } + print_peer_socket_info(peer); + if (!data || len == 0) { + wg_dbg("Exiting function wg_tcp_queuepkt, invalid parameters\n"); + return -EINVAL; + } + + /* Print TCP-related flags */ + wg_dbg("wg_peer: temp_peer = %d\n", peer->temp_peer); + wg_dbg("wg_peer: tcp_established = %d\n", peer->tcp_established); + wg_dbg("wg_peer: tcp_pending = %d\n", peer->tcp_pending); + wg_dbg("wg_peer: outbound_connected = %d\n", peer->outbound_connected); + wg_dbg("wg_peer: inbound_connected = %d\n", peer->inbound_connected); + wg_dbg("wg_peer: tcp_outbound_callbacks_set = %d\n", peer->tcp_outbound_callbacks_set); + wg_dbg("wg_peer: tcp_inbound_callbacks_set = %d\n", peer->tcp_inbound_callbacks_set); + log_wireguard_endpoint(&peer->endpoint); + + peer = wg_find_peer_by_endpoints(peer->device, &peer->endpoint); + if (!peer || IS_ERR(peer)) { + wg_dbg("wg_queuepkt: No matching peer found for endpoint\n"); + return -ENOENT; + } + + skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + if (!skb) { + wg_dbg("Exiting function wg_tcp_queuepkt\n"); + return -ENOMEM; + } + + skb_reserve(skb, SKB_HEADER_LEN); + skb_put_data(skb, data, len); + memset(skb->cb, 0, sizeof(skb->cb)); + + /* Diagnostic: Print packet details and check for fragmentation markers */ + wg_dbg("wg_tcp_queuepkt: Created skb=%px, len=%zu, skb->len=%u," + "skb->data_len=%u\n", skb, len, skb->len, skb->data_len); + wg_dbg("wg_tcp_queuepkt: First 32 bytes: %*ph\n", + min_t(int, skb->len, 32), skb->data); /* BUG FIX: was %*px (pointer with width) not %*ph (hex dump) */ + + /* Check if this looks like a fragmented packet (look for potential markers) */ + if (skb->len >= 4) { + __be32 *potential_frag_header = (__be32 *)skb->data; + wg_dbg("wg_tcp_queuepkt: Potential frag header: " + "0x%08x\n", ntohl(*potential_frag_header)); + } + + /* If this packet will get a TCP encap header, show what we expect */ + wg_dbg("wg_tcp_queuepkt: Expected TCP encap header length " + "will be: %zu + %zu = %zu\n", len, WG_TCP_ENCAP_HDR_LEN, + len + WG_TCP_ENCAP_HDR_LEN); + + frame = wg_tcp_build_frame(skb); + kfree_skb(skb); + if (IS_ERR(frame)) + return PTR_ERR(frame); + skb = frame; + + if (!peer->peer_socket) { + /* peer connenction is down reconnect */ + if (wg_tcp_connect(peer) < 0) { + kfree_skb(skb); + wg_dbg("Exiting function wg_tcp_queuepkt due to connection failure\n"); + return -ECONNREFUSED; /* Connection attempt failed */ + } + } + + wg_dbg("Current endpoint:"); + log_wireguard_endpoint(¤t_endpoint); + wg_dbg("Peer endpoint:"); + log_wireguard_endpoint(&peer->endpoint); + wg_dbg("Peer peer_endpoint:"); + log_wireguard_endpoint(&peer->peer_endpoint); + + if (!peer->tcp_established) { + /* peer connenction is down reconnect */ + if (wg_tcp_connect(peer) < 0) { + kfree_skb(skb); + wg_dbg("Exiting function wg_tcp_queuepkt due to connection failure\n"); + return -ECONNREFUSED; /* Connection attempt failed */ + } + } + ret = wg_tcp_enqueue_frame(peer, skb); + print_peer_socket_info(peer); + wg_dbg("Exiting function wg_tcp_queuepkt\n"); + return ret; +} + +/* Simple checksum function for TCP encapsulation header */ +static __be16 wg_header_checksum(const struct wg_tcp_encap_header *hdr) +{ + wg_dbg("Entering function wg_header_checksum\n"); + uint16_t checksum = 0; + uint32_t length = ntohl(hdr->length); + + checksum ^= (length >> 16) & 0xFFFF; + checksum ^= length & 0xFFFF; + checksum ^= (hdr->flags << 8) | hdr->type; + + checksum = (checksum << 5) | (checksum >> (16 - 5)); + + /* Avoid trivial all-zero or all-one checksums. */ + const uint16_t constant = 0xA5A5; + checksum ^= constant; + + wg_dbg("Exiting function wg_header_checksum\n"); + return htons(checksum); +} + +/* Function to validate the header checksum */ +static bool wg_validate_header_checksum(const struct wg_tcp_encap_header *hdr) +{ + wg_dbg("Entering function wg_validate_header_checksum\n"); + wg_dbg("Exiting function wg_validate_header_checksum\n"); + return wg_header_checksum(hdr) == hdr->checksum; +} + + +static int wg_tcp_send_frame(struct wg_peer *peer, struct socket *sock, + const struct sk_buff *frame) +{ + size_t send_len = wg_tcp_test_send_len(frame->len); + struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; + struct kvec vec = { + .iov_base = (void *)frame->data, + .iov_len = send_len + }; + int sent; + +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sock->sk, "tx:frame:pre", 0, frame->len); +#endif + if (wg_tcp_test_take_fatal_send(peer, sock)) + sent = -EPIPE; + else + sent = kernel_sendmsg(sock, &msg, &vec, 1, send_len); +#if defined(DEBUG) && defined(WG_TCP_FAULT_INJECTION) + if (sent > 0 && (unsigned int)sent < frame->len) + atomic64_inc(&wg_tcp_test_short_writes); +#endif +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sock->sk, "tx:frame:post", sent, frame->len); + if (sent > 0) + atomic64_add(sent, &wg_tcp_stats_tx_bytes); + if (sent >= 0 && (unsigned int)sent < frame->len) + atomic64_inc(&wg_tcp_stats_short_writes); +#endif + return sent; +} + +static void wg_tcp_arm_write_space(struct socket *socket) +{ + set_bit(SOCK_NOSPACE, &socket->flags); + /* Pair with the writeability recheck before the worker releases its + * scheduled claim, as tcp_poll() does when arming EPOLLOUT. + */ + smp_mb__after_atomic(); +} + +static void wg_tcp_fail_exact_socket(struct wg_peer *peer, + struct socket *socket) +{ + struct wg_device *cleanup_device = NULL; + bool queue_outbound_remove = false; + bool queue_temp_cleanup = false; + + if (!peer || IS_ERR(peer) || !peer->device || !socket) + return; + spin_lock_bh(&peer->tcp_lock); + if (peer->peer_socket != socket) + goto unlock; + if (peer->temp_peer) { + if (peer->inbound_socket != socket) + goto unlock; + WRITE_ONCE(peer->is_dead, true); + cleanup_device = peer->device; + queue_temp_cleanup = + READ_ONCE(cleanup_device->tcp_cleanup_scheduled); + goto unlock; + } + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP || + peer->outbound_socket != socket) + goto unlock; + if (peer->tcp_outbound_remove_scheduled && + peer->tcp_outbound_remove_socket != socket) + goto unlock; + peer->outbound_timestamp = ktime_set(0, 0); + peer->outbound_connected = false; + peer->tcp_pending = false; + peer->tcp_established = false; + peer->tcp_reconnect_requested = true; + if (!peer->tcp_connecting && !peer->tcp_outbound_remove_scheduled) { + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = socket; + queue_outbound_remove = true; + } + if (queue_outbound_remove) + mod_delayed_work(system_wq, &peer->tcp_outbound_remove_work, 0); +unlock: + spin_unlock_bh(&peer->tcp_lock); + if (queue_temp_cleanup && + READ_ONCE(cleanup_device->tcp_cleanup_scheduled)) + mod_delayed_work(system_wq, &cleanup_device->tcp_cleanup_work, 0); +} + +void wg_tcp_write_worker(struct work_struct *work) +{ + + struct wg_peer *peer = container_of(work, struct wg_peer, tcp_write_work); + struct socket *socket = NULL; + struct sock *sk = NULL; + struct sk_buff *skb; + unsigned int write_delay_ms; + int sent; + + wg_dbg("Entering function wg_tcp_write_worker\n"); + + if (!peer) { + wg_dbg("wg_tcp_write_worker: Invalid peer or socket\n"); + goto out; + } + /* A remover sets its direction flag under tcp_lock before calling + * cancel_work_sync(). Once captured here, the socket therefore remains + * alive until this worker returns. + */ + spin_lock_bh(&peer->tcp_lock); + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + !peer->tcp_outbound_remove_scheduled && + !peer->tcp_inbound_remove_scheduled && peer->peer_socket && + peer->tcp_established) { + socket = peer->peer_socket; + sk = socket->sk; + } + spin_unlock_bh(&peer->tcp_lock); + if (!socket || !sk) { + wg_dbg("wg_tcp_write_worker: Socket is being removed\n"); + goto out; + } +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "tx:write_worker:start", 0, skb_queue_len(&peer->send_queue)); +#endif + wg_dbg("wg_tcp_write_worker: start peer=%llu send_queue_len=%u\n", + peer->internal_id, skb_queue_len(&peer->send_queue)); + + /* A single bounded DEBUG delay lets the fully counted queue fill without + * making teardown wait once per queued frame. Recheck ownership after the + * sleep because a remover can publish its stop flag while waiting for this + * worker to return. + */ + write_delay_ms = wg_tcp_test_take_write_delay_ms(); + if (write_delay_ms) { + msleep(write_delay_ms); + spin_lock_bh(&peer->tcp_lock); + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->tcp_outbound_remove_scheduled || + peer->tcp_inbound_remove_scheduled || + peer->peer_socket != socket || !peer->tcp_established) + socket = NULL; + spin_unlock_bh(&peer->tcp_lock); + if (!socket) + goto out; + } + + /* BUG FIX: dequeue under lock, send outside lock. + * kernel_sendmsg() calls lock_sock() which can sleep — + * must NOT hold a spinlock across it. + * + * Do not gate the send on sk_stream_is_writeable(). A nonblocking send + * that reaches EAGAIN arms SOCK_NOSPACE inside the stream layer, which is + * what makes the later write-space callback reliable. + */ + while (true) { + spin_lock_bh(&peer->send_queue_lock); + skb = __skb_dequeue(&peer->send_queue); + spin_unlock_bh(&peer->send_queue_lock); + + if (!skb) + break; + + /* The skb already contains the complete stream frame. A short write + * advances that exact byte sequence; it must never be reframed. + */ + sent = wg_tcp_send_frame(peer, socket, skb); + if (sent > 0) { + if ((unsigned int)sent > skb->len) { + pr_err("wg_tcp_write_worker: invalid write count %d/%u\n", + sent, skb->len); + kfree_skb(skb); + wg_tcp_fail_exact_socket(peer, socket); + break; + } + skb_pull(skb, sent); + if (skb->len) { +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "tx:write_worker:partial", + sent, skb->len); +#endif + spin_lock_bh(&peer->send_queue_lock); + __skb_queue_head(&peer->send_queue, skb); + spin_unlock_bh(&peer->send_queue_lock); + wg_tcp_arm_write_space(socket); + break; + } +#if WG_TCP_DIAG_ENABLED + atomic64_inc(&wg_tcp_stats_tx_packets); +#endif + kfree_skb(skb); + } else if (sent == -EAGAIN || sent == -EWOULDBLOCK) { +#if WG_TCP_DIAG_ENABLED + if (sent == -EAGAIN || sent == -EWOULDBLOCK) + atomic64_inc(&wg_tcp_stats_tx_eagain); + wg_tcp_diag_pressure(sk, peer->internal_id); +#endif + spin_lock_bh(&peer->send_queue_lock); + __skb_queue_head(&peer->send_queue, skb); + spin_unlock_bh(&peer->send_queue_lock); + wg_tcp_arm_write_space(socket); + break; + } else { + pr_debug_ratelimited("WireGuard: terminal TCP send error=%d peer=%llu frame_len=%u\n", + sent, peer->internal_id, skb->len); +#if defined(DEBUG) && defined(WG_TCP_FAULT_INJECTION) + atomic64_inc(&wg_tcp_test_fatal_send_errors); +#endif +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "tx:write_worker:error", sent, + skb->len); + atomic64_inc(&wg_tcp_stats_tx_errors); +#endif + kfree_skb(skb); + wg_tcp_fail_exact_socket(peer, socket); + break; + } + } + +out: + /* Clear and, if needed, reclaim the writer atomically with respect to + * producers and socket removal. A producer blocked on these locks will + * observe the cleared flag and queue the work itself. + */ + spin_lock_bh(&peer->tcp_lock); + spin_lock(&peer->tcp_write_lock); + peer->tcp_write_worker_scheduled = false; + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + !peer->tcp_outbound_remove_scheduled && + !peer->tcp_inbound_remove_scheduled && socket && sk && + peer->peer_socket == socket && peer->tcp_established && + peer->tcp_write_wq && skb_queue_len(&peer->send_queue) > 0 && + sk_stream_is_writeable(sk)) { + peer->tcp_write_worker_scheduled = true; + queue_work(peer->tcp_write_wq, &peer->tcp_write_work); + } + spin_unlock(&peer->tcp_write_lock); + spin_unlock_bh(&peer->tcp_lock); +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_aggregate(); /* Periodic aggregate stats */ +#endif + wg_dbg("Exiting function wg_tcp_write_work\n"); +} + +void wg_peer_discard_partial_read(struct wg_peer *peer); + +void wg_peer_discard_partial_read(struct wg_peer *peer) +{ + if (peer->partial_skb) + kfree_skb(peer->partial_skb); + peer->partial_skb = NULL; + peer->expected_len = 0; + peer->received_len = 0; +} + +bool wg_sync_header(struct wg_peer *peer, struct socket *socket); + +bool wg_sync_header(struct wg_peer *peer, struct socket *socket) +{ + size_t i, suffix_len; + + if (!peer || !socket || !socket->sk) + return false; + + wg_dbg("Entering function wg_sync_header\n"); + wg_dbg("wg_sync_header: Trying to synchonize to new header.\n"); + if (!peer->partial_skb || + peer->received_len < WG_TCP_ENCAP_HDR_LEN) + return false; + + for (i = 0; i <= peer->received_len - WG_TCP_ENCAP_HDR_LEN; ++i) { + struct wg_tcp_encap_header *potential_hdr = + (struct wg_tcp_encap_header *)(peer->partial_skb->data + i); + struct wg_tcp_encap_header candidate; + + if (wg_check_potential_header_validity(potential_hdr, + peer->received_len - i)) { + memcpy(&candidate, potential_hdr, sizeof(candidate)); + wg_dbg("wg_sync_header: Found new header.\n"); + skb_pull(peer->partial_skb, i); + peer->received_len -= i; + peer->expected_len = ntohl(candidate.length); + wg_dbg("Exiting function wg_sync_header\n"); + return true; + } + } + + /* No complete candidate exists yet. Preserve the maximum possible header + * prefix so the next ordinary reader invocation can append bytes from a + * later TCP segment. Keeping fewer than a full header also prevents the + * worker from spinning on known-invalid data. + */ + suffix_len = min_t(size_t, peer->received_len, + WG_TCP_ENCAP_HDR_LEN - 1); + memmove(peer->partial_skb->data, + peer->partial_skb->data + peer->received_len - suffix_len, + suffix_len); + skb_trim(peer->partial_skb, suffix_len); + peer->received_len = suffix_len; + peer->expected_len = 0; + wg_dbg("Exiting function wg_sync_header\n"); + return false; +} + +/* Function to check if the given data pointer has a valid WireGuard TCP encapsulation header */ +bool wg_check_potential_header_validity(struct wg_tcp_encap_header *hdr, size_t remaining_len) +{ + struct wg_tcp_encap_header candidate; + size_t minimum_len = WG_TCP_ENCAP_HDR_LEN + MESSAGE_MINIMUM_LENGTH; + u32 total_len; + + if (remaining_len < WG_TCP_ENCAP_HDR_LEN) + return false; + memcpy(&candidate, hdr, sizeof(candidate)); + if (!wg_validate_header_checksum(&candidate)) + return false; + if (candidate.type != WG_TCP_RECORD_DATA) + return false; + if (candidate.flags & ~WG_TCP_FRAG_FLAG) + return false; + if (candidate.flags & WG_TCP_FRAG_FLAG) + minimum_len += WG_TCP_FRAG_HDR_LEN; + total_len = ntohl(candidate.length); + return total_len >= minimum_len && total_len <= WG_MAX_PACKET_SIZE; +} + +static int wg_tcp_build_fake_headers(struct sk_buff *skb, struct wg_peer *peer, + struct socket *socket) +{ + struct iphdr *iph; + struct udphdr *udph; + struct sock *sk; + struct inet_sock *inet; + struct sockaddr_in outbound_source, outbound_dest; + int payload_len; +#if IS_ENABLED(CONFIG_IPV6) + struct sockaddr_in6 outbound_source6, outbound_dest6; +#endif + + /* Diagnostic: Print SKB state on entry */ + wg_dbg("Entering wg_tcp_build_fake_headers. SKB state on entry: " + "skb=%px, len=%d, head=%px, data=%px, tail=%u, end=%u, headroom=%d, tailroom=%d\n", + skb, skb->len, skb->head, skb->data, skb->tail, skb->end, skb_headroom(skb), skb_tailroom(skb)); + + log_wireguard_endpoint(&peer->endpoint); + + /* Initialize address pointers */ + struct sockaddr_in *source = NULL; + struct sockaddr_in *dest = NULL; +#if IS_ENABLED(CONFIG_IPV6) + struct sockaddr_in6 *source6 = NULL; + struct sockaddr_in6 *dest6 = NULL; +#endif + if (!socket || !socket->sk) + return -ENOTCONN; + sk = socket->sk; + + /* Use the socket pinned by the reader. For outbound streams, derive the + * tuple from the connected socket so route-selected source addresses and + * ephemeral ports cannot be stale. + */ + if (socket == READ_ONCE(peer->inbound_socket)) { + if (peer->inbound_source.ss_family == AF_INET) { + source = (struct sockaddr_in *)&peer->inbound_dest; + dest = (struct sockaddr_in *)&peer->inbound_source; +#if IS_ENABLED(CONFIG_IPV6) + } else if (peer->inbound_source.ss_family == AF_INET6) { + source6 = (struct sockaddr_in6 *)&peer->inbound_dest; + dest6 = (struct sockaddr_in6 *)&peer->inbound_source; +#endif + } + } else if (sk->sk_family == AF_INET) { + inet = inet_sk(sk); + memset(&outbound_source, 0, sizeof(outbound_source)); + memset(&outbound_dest, 0, sizeof(outbound_dest)); + outbound_source.sin_family = AF_INET; + outbound_source.sin_port = inet->inet_sport; + outbound_source.sin_addr.s_addr = inet->inet_saddr; + outbound_dest.sin_family = AF_INET; + outbound_dest.sin_port = inet->inet_dport; + outbound_dest.sin_addr.s_addr = inet->inet_daddr; + source = &outbound_dest; + dest = &outbound_source; +#if IS_ENABLED(CONFIG_IPV6) + } else if (sk->sk_family == AF_INET6) { + inet = inet_sk(sk); + memset(&outbound_source6, 0, sizeof(outbound_source6)); + memset(&outbound_dest6, 0, sizeof(outbound_dest6)); + outbound_source6.sin6_family = AF_INET6; + outbound_source6.sin6_port = inet->inet_sport; + outbound_source6.sin6_addr = inet6_sk(sk)->saddr; + outbound_dest6.sin6_family = AF_INET6; + outbound_dest6.sin6_port = inet->inet_dport; + outbound_dest6.sin6_addr = sk->sk_v6_daddr; + source6 = &outbound_dest6; + dest6 = &outbound_source6; +#endif + } else { + return -EAFNOSUPPORT; + } + + /* Check for paged data in the skb before forcibly linearizing it */ + if (skb_is_nonlinear(skb)) { + if (skb_linearize(skb) != 0) { + printk(KERN_ERR "wg_tcp_build_fake_headers: Failed to linearize SKB.\n"); + return -ENOMEM; + } else { + skb_reset_tail_pointer(skb); + } + } + + /* Diagnostic: Print SKB state after linearization */ + wg_dbg("After skb_linearize: skb=%px, len=%d, head=%px, data=%px, tail=%u, end=%u, skb->len=%d, headroom=%d, tailroom=%d\n", + skb, skb->len, skb->head, skb->data, skb->tail, skb->end, skb->len, skb_headroom(skb), skb_tailroom(skb)); + + /* Calculate the payload length: initial length of skb before any header is added */ + payload_len = skb->len; + + /* Push and reset for UDP header */ + skb_push(skb, sizeof(struct udphdr)); + skb_reset_transport_header(skb); + + /* Diagnostic: Print UDP header location */ + wg_dbg("UDP header location: %px, length: %zu\n", + skb_transport_header(skb), sizeof(struct udphdr)); + + /* Push and reset for IP header */ + if (source) { + skb_push(skb, sizeof(struct iphdr)); + skb_reset_network_header(skb); + wg_dbg("IPv4 header location: %px, length: %zu\n", + skb_network_header(skb), sizeof(struct iphdr)); +#if IS_ENABLED(CONFIG_IPV6) + } else if (source6) { + skb_push(skb, sizeof(struct ipv6hdr)); + skb_reset_network_header(skb); + wg_dbg("IPv6 header location: %px, length: %zu\n", + skb_network_header(skb), sizeof(struct ipv6hdr)); +#endif + } else { + printk(KERN_ERR "wg_tcp_build_fake_headers: Unsupported address family.\n"); + return -EAFNOSUPPORT; + } + + /* Diagnostic: Print SKB state after header manipulation */ + wg_dbg("After header manipulation: skb=%px, len=%d, head=%px, data=%px, tail=%u, end=%u, skb->len=%d, headroom=%d, tailroom=%d\n", + skb, skb->len, skb->head, skb->data, skb->tail, skb->end, skb->len, skb_headroom(skb), skb_tailroom(skb)); + + /* Set UDP header fields */ + udph = udp_hdr(skb); + if (source) { /* IPv4 case */ + udph->source = source->sin_port; + udph->dest = dest->sin_port; +#if IS_ENABLED(CONFIG_IPV6) + } else if (source6) { /* IPv6 case */ + udph->source = source6->sin6_port; + udph->dest = dest6->sin6_port; +#endif + } + udph->len = htons(sizeof(struct udphdr) + payload_len); + udph->check = 0; /* Checksum will be calculated later */ + + if (source) { + /* Fill in the IPv4 header */ + iph = ip_hdr(skb); + iph->version = 4; + iph->ihl = 5; + iph->tos = 0; + iph->tot_len = htons(sizeof(struct iphdr) + sizeof(struct udphdr) + payload_len); + iph->ttl = 64; + iph->protocol = IPPROTO_UDP; + iph->check = 0; + iph->saddr = source->sin_addr.s_addr; + iph->daddr = dest->sin_addr.s_addr; + + /* Calculate IP checksum */ + iph->check = ip_fast_csum((u8 *)iph, iph->ihl); + + /* Calculate UDP checksum for IPv4 */ + __wsum csum = csum_partial(udph, ntohs(udph->len), 0); + udph->check = htons(csum_tcpudp_magic(iph->saddr, iph->daddr, udph->len, IPPROTO_UDP, csum)); + if (udph->check == 0) + udph->check = CSUM_MANGLED_0; + + skb->protocol = htons(ETH_P_IP); +#if IS_ENABLED(CONFIG_IPV6) + } else if (source6) { + struct ipv6hdr *ip6h = ipv6_hdr(skb); + + /* Fill in the IPv6 header */ + ip6h->version = 6; + ip6h->priority = 0; + memset(ip6h->flow_lbl, 0, sizeof(ip6h->flow_lbl)); + ip6h->payload_len = htons(sizeof(struct udphdr) + payload_len); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = 64; + ip6h->saddr = source6->sin6_addr; + ip6h->daddr = dest6->sin6_addr; + + /* Calculate UDP checksum for IPv6 */ + __wsum csum = csum_partial(udph, ntohs(udph->len), 0); + csum = csum_partial(&ip6h->saddr, sizeof(struct in6_addr), csum); + csum = csum_partial(&ip6h->daddr, sizeof(struct in6_addr), csum); + csum = csum_add(csum, htons(ntohs(udph->len))); + csum = csum_add(csum, htons(IPPROTO_UDP)); + + udph->check = csum_fold(csum); + if (udph->check == 0) + udph->check = CSUM_MANGLED_0; + + /* endpoint_from_skb derives a link-local scope from skb_iif. Carry + * the accepted/dialed socket's scope through the synthetic datagram so + * authenticated roaming does not replace a scoped target with scope 0. + */ + skb->skb_iif = source6->sin6_scope_id; + if (!skb->skb_iif) + skb->skb_iif = READ_ONCE(sk->sk_bound_dev_if); + skb->protocol = htons(ETH_P_IPV6); +#endif + } else { + printk(KERN_ERR "wg_tcp_build_fake_headers: Unsupported address family.\n"); + return -EAFNOSUPPORT; + } + + /* Diagnostic: Print SKB state after header manipulation */ + wg_dbg("After header pull on exit: skb=%px, len=%d, head=%px, data=%px, tail=%u, end=%u, headroom=%d, tailroom=%d\n", + skb, skb->len, skb->head, skb->data, skb->tail, skb->end, skb_headroom(skb), skb_tailroom(skb)); + + return 0; +} + + + +void wg_tcp_read_worker(struct work_struct *work) +{ + + wg_dbg("Entering function wg_tcp_read_worker\n"); + struct wg_tcp_frag_header frag_hdr; + bool has_frag_header = false; + struct wg_peer *peer = container_of(work, struct wg_peer, tcp_read_work); + struct socket *socket = NULL; + struct sock *sk; + struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; + struct kvec vec; + size_t packet_header_length; + ssize_t read_bytes; + unsigned int packets_processed = 0; + bool budget_exhausted = false; + struct sk_buff *new_skb = NULL; + + /* Pin the selected socket while holding the same lifetime lock used by + * removers. Once a remover publishes its flag, cancel_work_sync() keeps + * this socket alive until the reader returns. + */ + if (!peer || IS_ERR(peer)) + goto out; + spin_lock_bh(&peer->tcp_lock); + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + !peer->tcp_outbound_remove_scheduled && + !peer->tcp_inbound_remove_scheduled && peer->tcp_established && + peer->peer_socket && peer->peer_socket->sk) { + socket = peer->peer_socket; + sk = socket->sk; + } + spin_unlock_bh(&peer->tcp_lock); + if (!socket) + goto out; + print_peer_socket_info(peer); + while (true) { + bool record_ready; + + wg_dbg("wg_peer diagnostic: partial_skb=%px, expected_len=%zu, received_len=%zu\n", + peer->partial_skb, peer->expected_len, peer->received_len); + if (!peer->partial_skb) { + wg_dbg("wg_tcp_read_worker: Allocating new skb.\n"); + /* + * Leave room for the maximum record and its synthetic + * network headers. + */ + new_skb = alloc_skb(WG_TCP_SKB_READ_ALLOC_SIZE + + WG_TCP_RESERVED_HEADER_SIZE + + NET_IP_ALIGN, + GFP_ATOMIC); + if (!new_skb) { + pr_err("WireGuard: Failed to allocate skb\n"); + break; + } + /* Reserve space for headers and align the data correctly */ + skb_reserve(new_skb, WG_TCP_RESERVED_HEADER_SIZE + NET_IP_ALIGN); + + peer->expected_len = 0; + peer->partial_skb = new_skb; + } + record_ready = peer->expected_len ? + peer->received_len >= peer->expected_len : + peer->received_len >= WG_TCP_ENCAP_HDR_LEN; + if (!record_ready) { + /* Make sure we have enough room for at least an encapsulation header */ + if (skb_tailroom(peer->partial_skb) < WG_TCP_ENCAP_HDR_LEN) { + wg_dbg("wg_tcp_read_worker: Reallocating skb to fit the encapsulation header.\n"); + new_skb = skb_copy_expand(peer->partial_skb, skb_headroom(peer->partial_skb), + WG_TCP_SKB_READ_ALLOC_SIZE + WG_TCP_RESERVED_HEADER_SIZE + NET_IP_ALIGN, + GFP_ATOMIC); + if (!new_skb) { + pr_err("WireGuard: Failed to reallocate skb\n"); + wg_peer_discard_partial_read(peer); + break; + } + /* Replace the old skb with the new one */ + kfree_skb(peer->partial_skb); + peer->partial_skb = new_skb; + } + /* + * Read as much data as fits into the skb buffer + * When reading more data, make sure to append after existing data + */ + vec.iov_base = skb_tail_pointer(peer->partial_skb); + vec.iov_len = skb_tailroom(peer->partial_skb); + if (!vec.iov_len) + break; + read_bytes = kernel_recvmsg(socket, &msg, &vec, 1, + vec.iov_len, msg.msg_flags); + if (read_bytes > 0) { +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "rx:recvmsg", read_bytes, vec.iov_len); +#endif +#if WG_TCP_DIAG_ENABLED + atomic64_add(read_bytes, &wg_tcp_stats_rx_bytes); +#endif + } + if (read_bytes <= 0) { + if (read_bytes == -EAGAIN) { + wg_dbg("wg_tcp_read_worker: No more data available (-EAGAIN).\n"); + break; /* No more data available, exit the loop */ + } else if (read_bytes == 0) { + wg_dbg("wg_tcp_read_worker: peer closed the TCP stream\n"); + wg_peer_discard_partial_read(peer); + break; + } else { + pr_err("wg_tcp_read_worker: kernel_recvmsg error=%zd peer=%llu received_len=%zu expected_len=%zu\n", + read_bytes, peer->internal_id, peer->received_len, peer->expected_len); +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "rx:read_worker:error", read_bytes, vec.iov_len); +#endif +#if WG_TCP_DIAG_ENABLED + atomic64_inc(&wg_tcp_stats_rx_errors); +#endif + wg_peer_discard_partial_read(peer); + break; + } + } + /* Keep negative lengths out of the %*ph field width. */ + wg_dbg("wg_tcp_read_worker: kernel_recvmsg read %zd bytes: %*ph\n", read_bytes, (int)read_bytes, vec.iov_base); + wg_dbg("wg_tcp_read_worker: Read %zd bytes, total " + "received_len=%zu, expected_len=%zu\n", read_bytes, + peer->received_len, peer->expected_len); + skb_put(peer->partial_skb, read_bytes); + peer->received_len += read_bytes; + } + /* check header */ + if (peer->received_len >= WG_TCP_ENCAP_HDR_LEN) { + struct wg_tcp_encap_header header; + + /* Complete header received, validate and prepare for packet data */ + wg_dbg("wg_tcp_read_worker: We have a header, let's check it.\n"); + memcpy(&header, peer->partial_skb->data, sizeof(header)); + + /* Enhanced header diagnostics */ + wg_dbg("wg_tcp_read_worker: Processing TCP Encap Header\n"); + wg_dbg("wg_tcp_read_worker: Raw header bytes: %*phN\n", + (int)WG_TCP_ENCAP_HDR_LEN, &header); + wg_dbg("wg_tcp_read_worker: Header fields - length=0x%08x (%u)," + " type=%u, flags=0x%02x, checksum=0x%04x\n", + header.length, ntohl(header.length), header.type, + header.flags, ntohs(header.checksum)); + wg_dbg("wg_tcp_read_worker: Expected total packet " + "size: %u bytes\n", ntohl(header.length)); + + if (!wg_check_potential_header_validity(&header, + peer->received_len)) { + pr_debug_ratelimited( + "WireGuard: Invalid TCP record header, attempting resynchronization\n"); + if (!wg_sync_header(peer, socket)) { + /* A bounded suffix may be an incomplete header split + * across recvmsg calls. Keep it for data_ready rather + * than treating normal stream segmentation as damage. + */ + if (peer->partial_skb && + peer->received_len < WG_TCP_ENCAP_HDR_LEN) + break; + pr_debug_ratelimited( + "WireGuard: No valid TCP record header found\n"); + wg_peer_discard_partial_read(peer); + break; + } +#if defined(DEBUG) && defined(WG_TCP_FAULT_INJECTION) + atomic64_inc(&wg_tcp_test_resyncs); +#endif + /* Resynchronization can pull, free, or replace partial_skb. + * Copy and validate the selected candidate again before use. + */ + if (!peer->partial_skb || + peer->received_len < WG_TCP_ENCAP_HDR_LEN) { + wg_peer_discard_partial_read(peer); + break; + } + memcpy(&header, peer->partial_skb->data, + sizeof(header)); + if (!wg_check_potential_header_validity( + &header, peer->received_len)) { + wg_peer_discard_partial_read(peer); + break; + } + } + peer->expected_len = ntohl(header.length); + wg_dbg("wg_tcp_read_worker: sk=%px hdr: total_len=%zu type=%u flags=0x%02x checksum=0x%04x received_len=%zu\n", + sk, peer->expected_len, header.type, header.flags, + ntohs(header.checksum), peer->received_len); +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "rx:hdr", peer->received_len, peer->expected_len); +#endif + /* Check for fragment header flag */ + if (header.flags & WG_TCP_FRAG_FLAG) { + has_frag_header = true; + packet_header_length = WG_TCP_ENCAP_HDR_LEN + WG_TCP_FRAG_HDR_LEN; + wg_dbg("wg_tcp_read_worker: Fragment header flag detected\n"); + } else { + has_frag_header = false; + packet_header_length = WG_TCP_ENCAP_HDR_LEN; + } + wg_dbg("wg_tcp_read_worker: Set expected_len=%zu " + "(includes %zu byte header)\n", peer->expected_len, + packet_header_length); + + } else { + /* not enough data */ + break; + } + wg_dbg("wg_tcp_read_worker: We have a header, let's process the packet body.\n"); + /* + * A read may contain the current record plus bytes from the + * next one. + */ + if (peer->received_len < peer->expected_len) { + size_t needed = peer->expected_len - peer->received_len; + + if (skb_tailroom(peer->partial_skb) < needed) { + wg_dbg("wg_tcp_read_worker: We need more data for a full packet expected len=%d received_len=%d\n", (int)peer->expected_len, (int)peer->received_len); + wg_dbg("wg_tcp_read_worker: Expanding buffer to fit whole packet.\n"); + struct sk_buff *resized_skb = skb_copy_expand(peer->partial_skb, + skb_headroom(peer->partial_skb), + needed, + GFP_ATOMIC); + if (!resized_skb) { + pr_err("WireGuard: Failed to resize skb\n"); + wg_peer_discard_partial_read(peer); + break; + } + if (peer->partial_skb) + kfree_skb(peer->partial_skb); + peer->partial_skb = resized_skb; + } + } + wg_dbg("Expected Length: %zu Received Length: %zu\n", peer->expected_len, peer->received_len); + wg_dbg("wg_tcp_read_worker: Packet complete check - Expected: %zu, Received: %zu\n", + peer->expected_len, peer->received_len); + + /* Enhanced diagnostics for complete packet */ + if (peer->received_len >= peer->expected_len) { + wg_dbg("wg_tcp_read_worker: Complete packet received, first 32 bytes: %*ph\n", min_t(int, peer->partial_skb->len, 32), peer->partial_skb->data); + + } + /* Check if we've received the complete packet now */ + if (peer->received_len >= peer->expected_len && peer->received_len > WG_TCP_ENCAP_HDR_LEN) { + wg_dbg("wg_tcp_read_worker: We have a complete packet.\n"); + + if (has_frag_header) { + __be32 *after_tcp_hdr = (__be32 *)(peer->partial_skb->data + WG_TCP_ENCAP_HDR_LEN); + wg_dbg("wg_tcp_read_worker: After TCP " + "header, next 4 bytes: 0x%08x\n", ntohl(*after_tcp_hdr)); + wg_dbg("wg_tcp_read_worker: After TCP header, " + "next 16 bytes: %*ph\n", + 16, peer->partial_skb->data + WG_TCP_ENCAP_HDR_LEN); /* BUG FIX: was missing width arg for %*ph */ + } + + if (has_frag_header) { + /* BUG FIX: check for encap + frag header combined length, + * not just frag header alone (frag follows encap) + */ + if (peer->received_len >= WG_TCP_ENCAP_HDR_LEN + WG_TCP_FRAG_HDR_LEN) { + memcpy(&frag_hdr, + peer->partial_skb->data + WG_TCP_ENCAP_HDR_LEN, + sizeof(frag_hdr)); + wg_dbg("wg_tcp_read_worker: Fragment header extracted - id=0x%04x, frag_off=0x%04x\n", + ntohs(frag_hdr.id), + ntohs(frag_hdr.frag_off)); + } else { + pr_err("wg_tcp_read_worker: Not enough data for fragment header\n"); + break; + } + } + + /* Remove the encapsulation header from the skb */ + skb_pull(peer->partial_skb, packet_header_length); + peer->received_len -= packet_header_length; + peer->expected_len -= packet_header_length; + + wg_dbg("wg_tcp_read_worker: After removing " + "encapsulation header - skb->len=%u, " + "received_len=%zu, expected_len=%zu\n", + peer->partial_skb->len, peer->received_len, + peer->expected_len); + + wg_dbg("Packet: %px\n", peer->partial_skb->data); + wg_dbg("partial_skb->len=%d received_len=%zu expected_len=%zu\n", peer->partial_skb->len, peer->received_len, peer->expected_len); + /* Check if the skb has a valid length */ + if (unlikely(peer->partial_skb->len <= 0)) { + pr_warn("wg_receive: Dropped packet with invalid length %d\n", peer->partial_skb->len); + wg_peer_discard_partial_read(peer); /* Reset for the next packet */ + break; + } + /* Calculate leftover data length */ + size_t leftover_len = peer->received_len - peer->expected_len; + struct sk_buff *leftover_skb = NULL; + if (leftover_len > 0) { + wg_dbg("wg_tcp_read_worker: Leftover data at " + "end of packet, leftover_len=%zu\n", leftover_len); + wg_dbg("wg_tcp_read_worker: Last %zu bytes of packet: %*ph\n", + leftover_len, min_t(int, (int)leftover_len, 64), + peer->partial_skb->data + peer->expected_len); + + leftover_skb = alloc_skb(leftover_len + + WG_TCP_RESERVED_HEADER_SIZE + + NET_IP_ALIGN, + GFP_ATOMIC); + if (!leftover_skb) { + pr_err("WireGuard: Failed to allocate leftover skb\n"); + break; + } + /* + * BUG FIX: only reserve header space, not the full alloc size, + * otherwise tailroom is zero and skb_put/copy overflows + */ + skb_reserve(leftover_skb, WG_TCP_RESERVED_HEADER_SIZE + + NET_IP_ALIGN); + + /* Diagnostic: Check skb pointers and lengths after skb_reserve */ + wg_dbg("wg_tcp_read_worker: leftover_skb after reserve: skb=%px, len=%d, headroom=%d, tailroom=%d\n", + leftover_skb, leftover_skb->len, skb_headroom(leftover_skb), skb_tailroom(leftover_skb)); + + /* + * BUG FIX: copy leftover data BEFORE trimming partial_skb, + * because skb_copy_bits fails when offset >= skb->len + * (skb_trim sets len = expected_len, making offset == len) + */ + if (skb_copy_bits(peer->partial_skb, peer->expected_len, leftover_skb->data, leftover_len) < 0) { + pr_err("wg_tcp_read_worker: Failed to copy leftover data (offset=%zu, skb->len=%u, copy_len=%zu)\n", + peer->expected_len, peer->partial_skb->len, leftover_len); + kfree_skb(leftover_skb); + leftover_skb = NULL; + wg_peer_discard_partial_read(peer); + break; + } + skb_put(leftover_skb, leftover_len); + + /* Now trim partial_skb after the copy is done */ + skb_trim(peer->partial_skb, peer->expected_len); + + wg_dbg("wg_tcp_read_worker: leftover_skb after copy, leftover_skb=%px, len=%d, headroom=%d, data=%px, tail=%u, end=%u\n", + leftover_skb, leftover_skb->len, skb_headroom(leftover_skb), leftover_skb->data, leftover_skb->tail, leftover_skb->end); + } + skb_set_tail_pointer(peer->partial_skb, peer->expected_len); + /* Store fragment info in packet_cb if we had a fragment header */ + if (has_frag_header) { + PACKET_CB(peer->partial_skb)->frag_id = + frag_hdr.id; + PACKET_CB(peer->partial_skb)->frag_off = + frag_hdr.frag_off; + } else { + PACKET_CB(peer->partial_skb)->frag_id = 0; + PACKET_CB(peer->partial_skb)->frag_off = 0; + } + + /* Build the UDP and IP headers */ + if (wg_tcp_build_fake_headers(peer->partial_skb, peer, + socket)) { + pr_err("WireGuard: Failed to build UDP/IP headers\n"); + wg_peer_discard_partial_read(peer); + break; + } + + /* Restore fragment fields if present */ + if (has_frag_header && peer->partial_skb->protocol == htons(ETH_P_IP)) { + struct iphdr *iph = ip_hdr(peer->partial_skb); + iph->id = frag_hdr.id; + iph->frag_off = frag_hdr.frag_off; + /* Recalculate IP checksum */ + iph->check = 0; + iph->check = ip_fast_csum((u8 *)iph, iph->ihl); + wg_dbg("wg_tcp_read_worker: Restored fragment fields to IP header\n"); + } + + /* Process the complete packet */ + wg_dbg("wg_tcp_read_worker: partial_skb after trim, partial_skb=%px, len=%d, head=%px, data=%px, tail=%u, end=%u\n", + peer->partial_skb, peer->partial_skb->len, peer->partial_skb->head, + peer->partial_skb->data, peer->partial_skb->tail, peer->partial_skb->end); + + wg_dbg("wg_tcp_read_worker: DELIVER sk=%px peer=%llu payload_len=%u wg_type=%u frag_id=%u frag_off=0x%x leftover_len=%zu\n", + sk, peer->internal_id, peer->partial_skb->len, + wg_tcp_diag_peek_msg_type(peer->partial_skb), + ntohs(PACKET_CB(peer->partial_skb)->frag_id), + ntohs(PACKET_CB(peer->partial_skb)->frag_off), + leftover_len); +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "rx:deliver", peer->partial_skb->len, peer->partial_skb->len); +#endif +#if WG_TCP_DIAG_ENABLED + atomic64_inc(&wg_tcp_stats_rx_packets); +#endif + wg_receive(sk, peer->partial_skb); /* wg_receive consumes the skb */ + + peer->partial_skb = NULL; /* wg_receive ate the data skb */ + if (leftover_len > 0) { + /* Store the leftover skb (if any) in peer->partial_skb */ + peer->partial_skb = leftover_skb; + peer->received_len = leftover_len; + + } else { + peer->received_len = 0; + } + peer->expected_len = 0; /* Reset for the next packet */ + } + if (++packets_processed >= 64) { + budget_exhausted = true; + break; + } + } +out: + /* Close the lost-wakeup window between the final nonblocking read and + * clearing the scheduled flag. data_ready uses the same lock, so either + * it queues the next worker or this worker observes pending receive data + * and queues itself again. tcp_lock is outermost, matching stream + * teardown, so no reader can be queued after a remover has claimed either + * socket and completed cancel_work_sync(). + */ + spin_lock_bh(&peer->tcp_lock); + spin_lock(&peer->tcp_read_lock); + peer->tcp_read_worker_scheduled = false; + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + !peer->tcp_outbound_remove_scheduled && + !peer->tcp_inbound_remove_scheduled && peer->tcp_read_wq && + socket && peer->peer_socket == socket && socket->sk && + (!skb_queue_empty(&socket->sk->sk_receive_queue) || + (budget_exhausted && peer->partial_skb && + !peer->expected_len && + peer->received_len >= WG_TCP_ENCAP_HDR_LEN))) { + peer->tcp_read_worker_scheduled = true; + queue_work(peer->tcp_read_wq, &peer->tcp_read_work); + } + spin_unlock(&peer->tcp_read_lock); + spin_unlock_bh(&peer->tcp_lock); + wg_dbg("Exiting function wg_tcp_read_worker\n"); +} + +void wg_tcp_data_ready(struct sock *sk) +{ + struct wg_socket_data *socket_data; + struct wg_peer *peer; + void (*original_data_ready)(struct sock *) = NULL; + + wg_dbg("Entering function wg_tcp_data_ready\n"); + + if (!sk || IS_ERR(sk)) { + printk(KERN_ERR "wg_tcp_data_ready: Invalid socket\n"); + goto done; + } + + read_lock_bh(&sk->sk_callback_lock); + socket_data = sk->sk_user_data; + + if (!socket_data || IS_ERR(socket_data)) { + printk(KERN_ERR "wg_tcp_data_ready: Invalid or NULL socket_data\n"); + goto unlock; + } + + peer = socket_data->peer; + if (!peer || IS_ERR(peer)) + goto unlock; + original_data_ready = socket_data->original_data_ready; + if (READ_ONCE(peer->is_dead)) + goto unlock; + if (peer->temp_peer) + wg_touch_tcp_connection(peer); + + + /* Match teardown's lifetime lock before taking the read scheduler lock. + * Queue while both are held so cancellation cannot miss newly claimed + * work after either socket removal has begun. + */ + spin_lock_bh(&peer->tcp_lock); + spin_lock(&peer->tcp_read_lock); + + /* Check if the worker is already scheduled and wq still exists */ + if (!READ_ONCE(peer->is_dead) && !peer->tcp_stopping && + READ_ONCE(peer->device->tcp_cleanup_scheduled) && + !peer->tcp_outbound_remove_scheduled && + !peer->tcp_inbound_remove_scheduled && + !peer->tcp_read_worker_scheduled && peer->tcp_read_wq) { + peer->tcp_read_worker_scheduled = true; +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "data_ready", 0, 0); +#endif + wg_dbg("wg_tcp_data_ready: schedule read worker peer=%llu sk=%px rcvq=%u\n", + peer->internal_id, sk, skb_queue_len(&sk->sk_receive_queue)); + queue_work(peer->tcp_read_wq, &peer->tcp_read_work); + } + + spin_unlock(&peer->tcp_read_lock); + spin_unlock_bh(&peer->tcp_lock); + +unlock: + if (original_data_ready) + original_data_ready(sk); + read_unlock_bh(&sk->sk_callback_lock); +done: + wg_dbg("Exiting function wg_tcp_data_ready\n"); +} + +void wg_tcp_write_space(struct sock *sk) +{ + struct wg_socket_data *socket_data; + struct wg_peer *peer; + void (*original_write_space)(struct sock *) = NULL; + + wg_dbg("Entering function wg_tcp_write_space\n"); + if (!sk || IS_ERR(sk)) + goto done; + + read_lock_bh(&sk->sk_callback_lock); + socket_data = sk->sk_user_data; + if (!socket_data || IS_ERR(socket_data)) + goto unlock; + peer = socket_data->peer; + if (!peer || IS_ERR(peer)) + goto unlock; + original_write_space = socket_data->original_write_space; + if (READ_ONCE(peer->is_dead)) + goto unlock; + if (!peer->tcp_write_wq) { + wg_dbg("wg_tcp_write_space peer->tcp_write_wq is NULL\n"); + goto unlock; + } + + wg_dbg("wg_tcp_write_space scheduling serial writer\n"); +#if WG_TCP_DIAG_ENABLED + wg_tcp_diag_dump_sock(sk, "write_space", 0, 0); +#endif + wg_dbg("wg_tcp_write_space: schedule write worker peer=%llu sk=%px writeq=%u\n", + peer->internal_id, sk, skb_queue_len(&sk->sk_write_queue)); + wg_tcp_schedule_write(peer); +unlock: + if (original_write_space) + original_write_space(sk); + read_unlock_bh(&sk->sk_callback_lock); +done: + wg_dbg("Exiting function wg_tcp_write_space\n"); +} + +static int wg_setup_tcp_socket_callbacks(struct wg_peer *peer, + struct socket *socket, bool inbound) +{ + struct wg_socket_data *socket_data; + struct wg_socket_data *installed, **owner; + struct sock *sk; + bool *callbacks_set; + int ret = 0; + + if (!peer || IS_ERR(peer) || !peer->device || !socket || !socket->sk) + return -EINVAL; + lockdep_assert_held(&peer->tcp_socket_lock); + + sk = socket->sk; + callbacks_set = inbound ? &peer->tcp_inbound_callbacks_set : + &peer->tcp_outbound_callbacks_set; + owner = inbound ? &peer->tcp_inbound_socket_data : + &peer->tcp_outbound_socket_data; + socket_data = kzalloc(sizeof(*socket_data), GFP_KERNEL); + if (!socket_data) + return -ENOMEM; + if (!try_module_get(THIS_MODULE)) { + kfree(socket_data); + return -ENODEV; + } + + write_lock_bh(&sk->sk_callback_lock); + spin_lock_bh(&peer->tcp_lock); + installed = sk->sk_user_data; + if ((inbound ? peer->inbound_socket : peer->outbound_socket) != socket || + READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP || + (inbound ? peer->tcp_inbound_remove_scheduled : + peer->tcp_outbound_remove_scheduled)) { + ret = -ESHUTDOWN; + goto unlock; + } + if (*callbacks_set) { + ret = *owner && installed == *owner && + (*owner)->peer == peer && (*owner)->socket == socket && + (*owner)->inbound == inbound && + sk->sk_state_change == wg_tcp_state_change && + sk->sk_write_space == wg_tcp_write_space && + sk->sk_data_ready == wg_tcp_data_ready ? 0 : -EUCLEAN; + goto unlock; + } + if (*owner) { + ret = -EUCLEAN; + goto unlock; + } + if (installed) { + ret = -EBUSY; + goto unlock; + } + + socket_data->device = peer->device; + socket_data->peer = peer; + socket_data->socket = socket; + socket_data->inbound = inbound; + socket_data->original_state_change = sk->sk_state_change; + socket_data->original_write_space = sk->sk_write_space; + socket_data->original_data_ready = sk->sk_data_ready; + sk->sk_user_data = socket_data; + *owner = socket_data; + sk->sk_state_change = wg_tcp_state_change; + sk->sk_write_space = wg_tcp_write_space; + sk->sk_data_ready = wg_tcp_data_ready; + *callbacks_set = true; + socket_data = NULL; +unlock: + spin_unlock_bh(&peer->tcp_lock); + write_unlock_bh(&sk->sk_callback_lock); + if (socket_data) { + module_put(THIS_MODULE); + kfree(socket_data); + } + return ret; +} + +static int wg_reset_tcp_socket_callbacks(struct wg_peer *peer, + struct socket *socket, bool inbound) +{ + struct wg_socket_data *socket_data = NULL, **owner; + struct sock *sk; + bool *callbacks_set; + int ret = 0; + + if (!peer || IS_ERR(peer) || !socket || !socket->sk) + return -EINVAL; + lockdep_assert_held(&peer->tcp_socket_lock); + + sk = socket->sk; + callbacks_set = inbound ? &peer->tcp_inbound_callbacks_set : + &peer->tcp_outbound_callbacks_set; + owner = inbound ? &peer->tcp_inbound_socket_data : + &peer->tcp_outbound_socket_data; + write_lock_bh(&sk->sk_callback_lock); + spin_lock_bh(&peer->tcp_lock); + if ((inbound ? peer->inbound_socket : peer->outbound_socket) != socket) { + ret = -ESTALE; + goto unlock; + } + if (!*callbacks_set) { + if (WARN_ON_ONCE(*owner || + sk->sk_state_change == wg_tcp_state_change || + sk->sk_write_space == wg_tcp_write_space || + sk->sk_data_ready == wg_tcp_data_ready)) + ret = -EUCLEAN; + goto unlock; + } + + socket_data = *owner; + if (WARN_ON_ONCE(!socket_data || socket_data->peer != peer || + socket_data->socket != socket || + socket_data->inbound != inbound)) { + ret = -EUCLEAN; + goto unlock; + } + if (sk->sk_state_change == wg_tcp_state_change) + sk->sk_state_change = socket_data->original_state_change; + if (sk->sk_write_space == wg_tcp_write_space) + sk->sk_write_space = socket_data->original_write_space; + if (sk->sk_data_ready == wg_tcp_data_ready) + sk->sk_data_ready = socket_data->original_data_ready; + if (sk->sk_user_data == socket_data) + sk->sk_user_data = NULL; + else + WARN_ON_ONCE(1); + *owner = NULL; + *callbacks_set = false; +unlock: + spin_unlock_bh(&peer->tcp_lock); + write_unlock_bh(&sk->sk_callback_lock); + if (!ret && socket_data) { + module_put(THIS_MODULE); + kfree(socket_data); + } + return ret; +} + +static int wg_reset_exact_tcp_socket_callbacks(struct wg_peer *peer, + struct socket *socket) +{ + bool inbound_alias, outbound_alias; + bool inbound_owned, outbound_owned; + + if (!peer || IS_ERR(peer) || !socket) + return -EINVAL; + lockdep_assert_held(&peer->tcp_socket_lock); + + spin_lock_bh(&peer->tcp_lock); + inbound_alias = peer->inbound_socket == socket; + outbound_alias = peer->outbound_socket == socket; + inbound_owned = inbound_alias && + (peer->tcp_inbound_callbacks_set || peer->tcp_inbound_socket_data); + outbound_owned = outbound_alias && + (peer->tcp_outbound_callbacks_set || peer->tcp_outbound_socket_data); + spin_unlock_bh(&peer->tcp_lock); + + if (!inbound_alias && !outbound_alias) + return -ESTALE; + if (WARN_ON_ONCE(inbound_owned && outbound_owned)) + return -EUCLEAN; + if (inbound_owned) + return wg_reset_tcp_socket_callbacks(peer, socket, true); + if (outbound_owned) + return wg_reset_tcp_socket_callbacks(peer, socket, false); + return wg_reset_tcp_socket_callbacks(peer, socket, !outbound_alias); +} + +void wg_tcp_retry_worker(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, tcp_retry_work.work); + bool queue_outbound_remove = false; + bool removal_pending; + int ret; + + wg_dbg("Entering function wg_tcp_retry_worker peer=%px\n", peer); + spin_lock_bh(&peer->tcp_lock); + peer->tcp_retry_scheduled = false; + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP) { + spin_unlock_bh(&peer->tcp_lock); + goto out; + } + if (!peer->tcp_established && peer->tcp_pending) { + /* Delegate destruction to the single outbound removal owner. It sets + * the lifetime flag before canceling stream work and releasing the + * socket, and reconnects after the old attempt is fully quiescent. + */ + peer->tcp_reconnect_requested = true; + if (!peer->tcp_outbound_remove_scheduled) { + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = peer->outbound_socket; + queue_outbound_remove = true; + } + } + if (queue_outbound_remove) + mod_delayed_work(system_wq, &peer->tcp_outbound_remove_work, 0); + removal_pending = peer->tcp_outbound_remove_scheduled; + spin_unlock_bh(&peer->tcp_lock); + if (removal_pending) + goto out; + + ret = wg_tcp_connect(peer); + if (ret < 0) { + spin_lock_bh(&peer->tcp_lock); + if (!peer->tcp_stopping && !peer->tcp_retry_scheduled) { + peer->tcp_retry_scheduled = true; + mod_delayed_work(system_wq, &peer->tcp_retry_work, + msecs_to_jiffies(30000)); + } + spin_unlock_bh(&peer->tcp_lock); + } + +out: + wg_dbg("Exiting function wg_tcp_retry_worker\n"); +} + +int wg_add_tcp_socket_to_list(struct wg_device *wg, struct socket *receive_socket, + struct wg_peer *temp_peer) +{ + wg_dbg("Entering function wg_add_tcp_socket_to_list\n"); + struct wg_tcp_socket_list_entry *entry; + struct wg_socket_data *socket_data; + struct sockaddr_storage addr; + int ret; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + pr_err("Failed to allocate wg_tcp_socket_list_entry\n"); + return -ENOMEM; + } + + entry->tcp_socket = receive_socket; + entry->temp_peer = temp_peer; /* BUG FIX: store temp_peer in list entry */ + socket_data = receive_socket && receive_socket->sk ? + READ_ONCE(receive_socket->sk->sk_user_data) : NULL; + if (!socket_data || socket_data->peer != temp_peer || + !socket_data->inbound) { + kfree(entry); + return -EINVAL; + } + entry->created_at = ktime_get(); + entry->timestamp = entry->created_at; + entry->connection_id = atomic64_inc_return( + &wg->tcp_connection_sequence); + entry->admission_counted = true; + entry->initializing = true; + temp_peer->tcp_connection_id = entry->connection_id; + + memset(&addr, 0, sizeof(addr)); + + ret = receive_socket->ops->getname(receive_socket, + (struct sockaddr *)&addr, 1); + if (ret < 0 || + !wg_sockaddr_length_valid((const struct sockaddr *)&addr, ret)) { + pr_err("Failed to get peer address from socket\n"); + kfree(entry); + return ret < 0 ? ret : -EINVAL; + } + if (!READ_ONCE(wg->tcp_cleanup_scheduled)) { + kfree(entry); + return -ESHUTDOWN; + } + + memcpy(&entry->src_addr, &addr, sizeof(addr)); + + spin_lock_bh(&wg->tcp_connection_list_lock); + if (!READ_ONCE(wg->tcp_cleanup_scheduled) || + wg->tcp_tracked_connections >= WG_TCP_MAX_TRACKED_CONNECTIONS || + wg->tcp_pending_connections >= WG_TCP_MAX_PENDING_CONNECTIONS || + wg_tcp_pending_from_source_locked( + wg, (const struct sockaddr *)&addr) >= + WG_TCP_MAX_PENDING_PER_SOURCE) { + spin_unlock_bh(&wg->tcp_connection_list_lock); + kfree(entry); + return -ENOSPC; + } + /* Serialize carrier publication with live device-mark refresh. Either + * this write observes the new mark, or the updater sees the published + * entry and applies it while holding the same list lock. + */ + if (receive_socket->sk) + WRITE_ONCE(receive_socket->sk->sk_mark, wg->fwmark); + list_add_tail_rcu(&entry->tcp_connection_ll, &wg->tcp_connection_list); + ++wg->tcp_pending_connections; + ++wg->tcp_tracked_connections; + spin_unlock_bh(&wg->tcp_connection_list_lock); + /* Run once immediately, then the worker keeps checking live provisional + * sockets until the list is empty. mod_delayed_work also closes the race + * with a worker that is just finishing an empty-list pass. + */ + mod_delayed_work(system_wq, &wg->tcp_cleanup_work, 0); + + wg_dbg("Exiting function wg_add_tcp_socket_to_list\n"); + return 0; +} + +static void wg_finish_tcp_connection_init(struct wg_device *wg, + struct socket *socket) +{ + struct wg_tcp_socket_list_entry *entry; + bool cleanup = false; + + spin_lock_bh(&wg->tcp_connection_list_lock); + list_for_each_entry(entry, &wg->tcp_connection_list, tcp_connection_ll) { + if (entry->tcp_socket != socket) + continue; + if (!socket->sk || + READ_ONCE(socket->sk->sk_state) != TCP_ESTABLISHED || + !entry->temp_peer || IS_ERR(entry->temp_peer) || + READ_ONCE(entry->temp_peer->is_dead)) { + if (entry->temp_peer && !IS_ERR(entry->temp_peer)) + WRITE_ONCE(entry->temp_peer->is_dead, true); + cleanup = true; + } + entry->initializing = false; + break; + } + spin_unlock_bh(&wg->tcp_connection_list_lock); + + if (cleanup && READ_ONCE(wg->tcp_cleanup_scheduled)) + mod_delayed_work(system_wq, &wg->tcp_cleanup_work, 0); +} + +static void wg_touch_tcp_connection(struct wg_peer *peer) +{ + struct wg_tcp_socket_list_entry *entry; + struct wg_device *wg; + + if (!peer || IS_ERR(peer) || !peer->temp_peer || !peer->device) + return; + wg = peer->device; + spin_lock_bh(&wg->tcp_connection_list_lock); + list_for_each_entry(entry, &wg->tcp_connection_list, tcp_connection_ll) { + if (entry->temp_peer == peer) { + entry->timestamp = ktime_get(); + break; + } + } + spin_unlock_bh(&wg->tcp_connection_list_lock); +} + +static struct wg_tcp_socket_list_entry * +wg_claim_tcp_connection(struct wg_device *wg, struct socket *pending_socket, + bool cleanup_only) +{ + struct wg_tcp_socket_list_entry *entry; + struct wg_tcp_socket_list_entry *claimed = NULL; + const ktime_t now = ktime_get(); + + spin_lock_bh(&wg->tcp_connection_list_lock); + list_for_each_entry(entry, &wg->tcp_connection_list, tcp_connection_ll) { + if (pending_socket && entry->tcp_socket != pending_socket) + continue; + if (cleanup_only && entry->initializing) + continue; + if (cleanup_only && entry->temp_peer && + !IS_ERR(entry->temp_peer) && + !READ_ONCE(entry->temp_peer->is_dead) && + (entry->authenticated || + (ktime_ms_delta(now, entry->timestamp) < + WG_TCP_AUTH_IDLE_TIMEOUT_MS && + ktime_ms_delta(now, entry->created_at) < + WG_TCP_AUTH_MAX_LIFETIME_MS))) + continue; + wg_tcp_release_admission_locked(wg, entry); + if (WARN_ON_ONCE(!wg->tcp_tracked_connections)) + wg->tcp_tracked_connections = 0; + else + --wg->tcp_tracked_connections; + list_del_rcu(&entry->tcp_connection_ll); + claimed = entry; + break; + } + spin_unlock_bh(&wg->tcp_connection_list_lock); + if (claimed) + synchronize_rcu(); + return claimed; +} + +static void wg_tcp_cancel_stream_workers(struct wg_peer *peer) +{ + cancel_work_sync(&peer->tcp_read_work); + cancel_work_sync(&peer->tcp_write_work); + spin_lock_bh(&peer->tcp_lock); + spin_lock(&peer->tcp_read_lock); + peer->tcp_read_worker_scheduled = false; + spin_unlock(&peer->tcp_read_lock); + spin_lock(&peer->tcp_write_lock); + peer->tcp_write_worker_scheduled = false; + spin_unlock(&peer->tcp_write_lock); + spin_unlock_bh(&peer->tcp_lock); +} + +static void wg_tcp_rearm_surviving_stream_locked(struct wg_peer *peer) +{ + struct socket *socket; + + lockdep_assert_held(&peer->tcp_socket_lock); + spin_lock_bh(&peer->tcp_lock); + socket = peer->peer_socket; + if (READ_ONCE(peer->is_dead) || peer->tcp_stopping || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->tcp_outbound_remove_scheduled || + peer->tcp_inbound_remove_scheduled || !peer->tcp_established || + !socket || !socket->sk) { + spin_unlock_bh(&peer->tcp_lock); + return; + } + spin_lock(&peer->tcp_read_lock); + if (!peer->tcp_read_worker_scheduled && peer->tcp_read_wq && + !skb_queue_empty(&socket->sk->sk_receive_queue)) { + peer->tcp_read_worker_scheduled = true; + queue_work(peer->tcp_read_wq, &peer->tcp_read_work); + } + spin_unlock(&peer->tcp_read_lock); + if (skb_queue_len(&peer->send_queue) > 0) + wg_tcp_schedule_write_locked(peer); + spin_unlock_bh(&peer->tcp_lock); +} + +static bool wg_tcp_promote_authenticated_carrier(struct wg_peer *peer, + u64 connection_id) +{ + struct wg_tcp_socket_list_entry *entry = NULL, *iter; + struct wg_peer *temp; + struct socket *socket, *old_inbound, *old_outbound; + bool temp_detached = false; + bool socket_transferred = false; + bool stale; + int ret = 0; + + if (!peer || IS_ERR(peer) || !connection_id || + peer->device->transport != WG_TRANSPORT_TCP) + return false; + + read_lock_bh(&peer->endpoint_lock); + stale = connection_id < peer->tcp_roaming_connection_id; + read_unlock_bh(&peer->endpoint_lock); + + spin_lock_bh(&peer->device->tcp_connection_list_lock); + list_for_each_entry(iter, &peer->device->tcp_connection_list, + tcp_connection_ll) { + if (iter->connection_id != connection_id) + continue; + iter->authenticated = true; + wg_tcp_release_admission_locked(peer->device, iter); + iter->timestamp = ktime_get(); + if (!stale && !iter->initializing) { + if (WARN_ON_ONCE(!peer->device->tcp_tracked_connections)) + peer->device->tcp_tracked_connections = 0; + else + --peer->device->tcp_tracked_connections; + list_del_rcu(&iter->tcp_connection_ll); + entry = iter; + } + break; + } + spin_unlock_bh(&peer->device->tcp_connection_list_lock); + if (!entry) + return false; + synchronize_rcu(); + + temp = entry->temp_peer; + socket = entry->tcp_socket; + if (!temp || IS_ERR(temp) || !socket || temp == peer) + goto fail_entry; + + /* Configured peers are always locked before provisional peers. No other + * path takes two socket-owner mutexes, making concurrent authenticated + * candidates serialize without an address-dependent lock order. + */ + mutex_lock(&peer->tcp_socket_lock); + mutex_lock(&temp->tcp_socket_lock); + write_lock_bh(&peer->endpoint_lock); + if (connection_id < peer->tcp_roaming_connection_id) { + write_unlock_bh(&peer->endpoint_lock); + ret = -ESTALE; + goto unlock; + } + peer->tcp_roaming_connection_id = connection_id; + write_unlock_bh(&peer->endpoint_lock); + + spin_lock_bh(&temp->tcp_lock); + if (temp->inbound_socket != socket || temp->peer_socket != socket || + temp->tcp_connection_id != connection_id) { + spin_unlock_bh(&temp->tcp_lock); + ret = -ESTALE; + goto unlock; + } + WRITE_ONCE(temp->is_dead, true); + temp->tcp_stopping = true; + temp->tcp_inbound_remove_scheduled = true; + temp->tcp_inbound_remove_socket = socket; + spin_unlock_bh(&temp->tcp_lock); + + if (socket->sk) { + write_lock_bh(&socket->sk->sk_callback_lock); + write_unlock_bh(&socket->sk->sk_callback_lock); + } + wg_tcp_cancel_stream_workers(temp); + ret = wg_reset_exact_tcp_socket_callbacks(temp, socket); + if (ret) + goto unlock; + spin_lock_bh(&temp->tcp_lock); + temp->peer_socket = NULL; + temp->inbound_socket = NULL; + temp->inbound_connected = false; + temp->tcp_established = false; + temp->tcp_inbound_remove_scheduled = false; + temp->tcp_inbound_remove_socket = NULL; + spin_unlock_bh(&temp->tcp_lock); + temp_detached = true; + + spin_lock_bh(&peer->tcp_lock); + old_inbound = peer->inbound_socket; + old_outbound = peer->outbound_socket; + peer->tcp_inbound_remove_scheduled = !!old_inbound; + peer->tcp_inbound_remove_socket = old_inbound; + peer->tcp_outbound_remove_scheduled = !!old_outbound; + peer->tcp_outbound_remove_socket = old_outbound; + spin_unlock_bh(&peer->tcp_lock); + if (old_inbound || old_outbound) + wg_tcp_cancel_stream_workers(peer); + if (old_inbound) { + ret = wg_reset_exact_tcp_socket_callbacks(peer, old_inbound); + if (!ret) + ret = wg_release_peer_socket_locked(peer, old_inbound); + if (ret) + goto unlock; + } + if (old_outbound && old_outbound != old_inbound) { + ret = wg_reset_exact_tcp_socket_callbacks(peer, old_outbound); + if (!ret) + ret = wg_release_peer_socket_locked(peer, old_outbound); + if (ret) + goto unlock; + } + + /* From this point the configured peer, rather than the claimed list + * entry, owns the accepted socket. + */ + entry->tcp_socket = NULL; + socket_transferred = true; + spin_lock_bh(&peer->tcp_lock); + /* The stream reader synthesizes the original outer IP/UDP headers for + * WireGuard's authenticated receive path. Preserve the accepted tuple + * when ownership moves away from the provisional peer. + */ + peer->inbound_source = temp->inbound_source; + peer->inbound_dest = temp->inbound_dest; + peer->peer_socket = socket; + peer->inbound_socket = socket; + peer->tcp_established = true; + peer->tcp_pending = false; + peer->tcp_connecting = false; + peer->inbound_connected = true; + peer->outbound_connected = false; + peer->inbound_timestamp = ktime_get(); + peer->tcp_inbound_remove_scheduled = false; + peer->tcp_inbound_remove_socket = NULL; + peer->tcp_outbound_remove_scheduled = false; + peer->tcp_outbound_remove_socket = NULL; + peer->tcp_reconnect_requested = false; + spin_unlock_bh(&peer->tcp_lock); + ret = wg_setup_tcp_socket_callbacks(peer, socket, true); + if (ret) { + spin_lock_bh(&peer->tcp_lock); + peer->tcp_inbound_callbacks_set = false; + peer->tcp_inbound_socket_data = NULL; + spin_unlock_bh(&peer->tcp_lock); + wg_release_peer_socket_locked(peer, socket); + goto unlock; + } + wg_tcp_rearm_surviving_stream_locked(peer); + +unlock: + mutex_unlock(&temp->tcp_socket_lock); + mutex_unlock(&peer->tcp_socket_lock); + if (ret) { + /* Concurrent candidates can become stale, and interface teardown can + * stop callback installation after a candidate has been claimed. Both + * paths safely destroy the candidate; warn only on unexpected errors. + */ + if (ret != -ESTALE && ret != -ESHUTDOWN) + WARN_ON_ONCE(ret); + goto fail_entry; + } + + temp->tcp_read_wq = NULL; + temp->tcp_write_wq = NULL; + if (temp->partial_skb) + kfree_skb(temp->partial_skb); + skb_queue_purge(&temp->send_queue); + entry->temp_peer = NULL; + entry->tcp_socket = NULL; + kfree(temp); + kfree(entry); + return true; + +fail_entry: + /* The list claim is exclusive. If handoff cannot complete, destroy the + * provisional owner rather than leaving an untracked accepted socket. + */ + if (temp_detached) { + temp->tcp_read_wq = NULL; + temp->tcp_write_wq = NULL; + if (temp->partial_skb) + kfree_skb(temp->partial_skb); + skb_queue_purge(&temp->send_queue); + entry->temp_peer = NULL; + kfree(temp); + } + /* A transferred socket is released by the configured-peer failure path. + * Leaving the entry pointer NULL prevents a second sock_release(). + */ + if (socket_transferred) + entry->tcp_socket = NULL; + wg_destroy_tcp_connection_entry(peer->device, entry); + return false; +} + +void wg_tcp_promotion_worker(struct work_struct *work) +{ + struct wg_peer *peer = + container_of(work, struct wg_peer, tcp_promotion_work); + u64 connection_id; + + /* Authenticated data is finalized from NAPI/softirq context. Claiming an + * accepted carrier uses synchronize_rcu(), mutexes, and workqueue drains, + * so perform the ownership transfer only from this process-context work. + */ + for (;;) { + spin_lock_bh(&peer->tcp_lock); + connection_id = peer->tcp_promotion_connection_id; + peer->tcp_promotion_connection_id = 0; + if (!connection_id) + peer->tcp_promotion_worker_scheduled = false; + spin_unlock_bh(&peer->tcp_lock); + if (!connection_id) + break; + wg_tcp_promote_authenticated_carrier(peer, connection_id); + } +} + +static void wg_destroy_temp_peer(struct wg_peer *peer) +{ + struct socket *socket; + struct sock *sk; + int ret = 0; + + if (!peer || IS_ERR(peer)) + return; + + spin_lock_bh(&peer->tcp_lock); + WRITE_ONCE(peer->is_dead, true); + peer->tcp_stopping = true; + peer->tcp_inbound_remove_scheduled = true; + peer->tcp_inbound_remove_socket = peer->inbound_socket; + peer->tcp_outbound_remove_scheduled = true; + peer->tcp_outbound_remove_socket = peer->outbound_socket; + spin_unlock_bh(&peer->tcp_lock); + cancel_delayed_work_sync(&peer->tcp_retry_work); + cancel_delayed_work_sync(&peer->tcp_outbound_remove_work); + cancel_delayed_work_sync(&peer->tcp_inbound_remove_work); + + mutex_lock(&peer->tcp_socket_lock); + socket = peer->inbound_socket; + sk = socket ? socket->sk : NULL; + /* Wait out a callback that passed the is_dead check before canceling + * work that may dereference sk_user_data. + */ + if (sk) { + write_lock_bh(&sk->sk_callback_lock); + write_unlock_bh(&sk->sk_callback_lock); + } + wg_tcp_cancel_stream_workers(peer); + + /* The workers are quiescent, so the wrapper can now be detached. */ + if (socket) + ret = wg_reset_exact_tcp_socket_callbacks(peer, socket); + /* Both pointers reference the device-scoped provisional queue. It remains + * alive until every pending entry has been drained during device teardown. + */ + if (!ret && socket) + ret = wg_release_peer_socket_locked(peer, socket); + if (!ret) { + peer->tcp_read_wq = NULL; + peer->tcp_write_wq = NULL; + } + mutex_unlock(&peer->tcp_socket_lock); + if (WARN_ON_ONCE(ret)) { + pr_err("WireGuard: retained provisional TCP peer after callback detach failure\n"); + return; + } + kfree(peer); +} + +static void +wg_destroy_tcp_connection_entry(struct wg_device *wg, + struct wg_tcp_socket_list_entry *entry) +{ + if (!entry) + return; + if (entry->temp_peer && !IS_ERR(entry->temp_peer)) { + wg_destroy_temp_peer(entry->temp_peer); + } else if (entry->tcp_socket) { + kernel_sock_shutdown(entry->tcp_socket, SHUT_RDWR); + sock_release(entry->tcp_socket); + } + kfree(entry); +} + +void wg_remove_from_tcp_connection_list(struct wg_device *wg, + struct socket *pending_socket) +{ + struct wg_tcp_socket_list_entry *entry; + + wg_dbg("Entering function wg_remove_from_tcp_connection_list\n"); + if (!wg || !pending_socket) + return; + entry = wg_claim_tcp_connection(wg, pending_socket, false); + wg_destroy_tcp_connection_entry(wg, entry); + wg_dbg("Exiting function wg_remove_from_tcp_connection_list\n"); +} + +void wg_tcp_outbound_remove_worker(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, tcp_outbound_remove_work.work); + struct socket *socket; + struct sock *sk; + bool active, detach_failed = false; + bool clean_claim, reclaim_current = false; + bool retry_needed, reconnect, stopping; + int ret; + + wg_dbg("Entering function wg_tcp_outbound_remove _worker\n"); + retry_needed = READ_ONCE(peer->tcp_retry_scheduled) || + delayed_work_pending(&peer->tcp_retry_work); + cancel_delayed_work_sync(&peer->tcp_retry_work); + mutex_lock(&peer->tcp_socket_lock); + spin_lock_bh(&peer->tcp_lock); + socket = peer->tcp_outbound_remove_socket; + clean_claim = socket ? peer->outbound_socket == socket : + !peer->outbound_socket; + sk = clean_claim && socket ? socket->sk : NULL; + active = clean_claim && socket && peer->peer_socket == socket; + peer->tcp_retry_scheduled = false; + spin_unlock_bh(&peer->tcp_lock); + + /* No new stream work is queued while the remove flag is set. Wait for + * callbacks that passed that check, then quiesce workers before freeing + * the sk_user_data wrapper. + */ + if (sk) { + write_lock_bh(&sk->sk_callback_lock); + write_unlock_bh(&sk->sk_callback_lock); + } + if (active) + wg_tcp_cancel_stream_workers(peer); + if (clean_claim) { + if (socket) { + ret = wg_reset_exact_tcp_socket_callbacks(peer, socket); + if (!ret) + ret = wg_release_peer_socket_locked(peer, socket); + if (ret) + detach_failed = true; + } + } else { + /* Never let an old work item tear down a replacement socket. */ + reclaim_current = true; + } + + spin_lock_bh(&peer->tcp_lock); + reconnect = peer->tcp_reconnect_requested; + stopping = peer->tcp_stopping; + if (!detach_failed) { + peer->tcp_reconnect_requested = false; + peer->tcp_outbound_remove_scheduled = false; + peer->tcp_outbound_remove_socket = NULL; + } + spin_unlock_bh(&peer->tcp_lock); + if (!detach_failed) + wg_tcp_rearm_surviving_stream_locked(peer); + mutex_unlock(&peer->tcp_socket_lock); + if (WARN_ON_ONCE(detach_failed)) { + pr_err("WireGuard: retained outbound TCP socket after callback detach failure\n"); + goto out; + } + + if (stopping || READ_ONCE(peer->is_dead) || + !READ_ONCE(peer->device->tcp_cleanup_scheduled) || + peer->device->transport != WG_TRANSPORT_TCP) + goto out; + if (reclaim_current) { + wg_tcp_peer_request_reconnect(peer); + goto out; + } + if (reconnect) { + ret = wg_tcp_connect(peer); + if (ret < 0) { + spin_lock_bh(&peer->tcp_lock); + if (!peer->tcp_stopping && + !peer->tcp_retry_scheduled) { + peer->tcp_retry_scheduled = true; + mod_delayed_work(system_wq, &peer->tcp_retry_work, + msecs_to_jiffies(30000)); + } + spin_unlock_bh(&peer->tcp_lock); + } + } else if (retry_needed) { + spin_lock_bh(&peer->tcp_lock); + if (!peer->tcp_stopping && !peer->tcp_retry_scheduled) { + peer->tcp_retry_scheduled = true; + mod_delayed_work(system_wq, &peer->tcp_retry_work, + msecs_to_jiffies(10000)); + } + spin_unlock_bh(&peer->tcp_lock); + } + +out: + wg_dbg("Exiting function wg_tcp_outbound_remove_worker\n"); +} + +void wg_tcp_inbound_remove_worker(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, tcp_inbound_remove_work.work); + struct socket *socket; + struct sock *sk; + bool active, clean_claim, detach_failed = false; + int ret = 0; + + wg_dbg("Entering function wg_tcp_inbound_remove _worker\n"); + + if (peer->temp_peer) { + WRITE_ONCE(peer->is_dead, true); + if (READ_ONCE(peer->device->tcp_cleanup_scheduled)) + mod_delayed_work(system_wq, + &peer->device->tcp_cleanup_work, 0); + } else { + mutex_lock(&peer->tcp_socket_lock); + spin_lock_bh(&peer->tcp_lock); + socket = peer->tcp_inbound_remove_socket; + clean_claim = socket ? peer->inbound_socket == socket : + !peer->inbound_socket; + sk = clean_claim && socket ? socket->sk : NULL; + active = clean_claim && socket && peer->peer_socket == socket; + spin_unlock_bh(&peer->tcp_lock); + if (sk) { + write_lock_bh(&sk->sk_callback_lock); + write_unlock_bh(&sk->sk_callback_lock); + } + if (active) + wg_tcp_cancel_stream_workers(peer); + if (clean_claim && socket) { + ret = wg_reset_exact_tcp_socket_callbacks(peer, socket); + if (!ret) + ret = wg_release_peer_socket_locked(peer, socket); + if (ret) + detach_failed = true; + } + spin_lock_bh(&peer->tcp_lock); + if (!detach_failed) { + peer->tcp_inbound_remove_scheduled = false; + peer->tcp_inbound_remove_socket = NULL; + } + spin_unlock_bh(&peer->tcp_lock); + if (!detach_failed) + wg_tcp_rearm_surviving_stream_locked(peer); + mutex_unlock(&peer->tcp_socket_lock); + if (WARN_ON_ONCE(detach_failed)) + pr_err("WireGuard: retained inbound TCP socket after callback detach failure\n"); + } + wg_dbg("Exiting function wg_inbound_remove_worker\n"); +} + +void wg_destruct_tcp_connection_list(struct wg_device *wg) +{ + struct wg_tcp_socket_list_entry *entry; + + wg_dbg("Entering function wg_destruct_tcp_connection_list\n"); + if (!wg) + return; + while ((entry = wg_claim_tcp_connection(wg, NULL, false))) + wg_destroy_tcp_connection_entry(wg, entry); + + wg_dbg("Exiting function wg_destruct_tcp_connection_list\n"); +} + +void wg_tcp_cleanup_worker(struct work_struct *work) +{ + struct wg_device *wg = container_of(work, struct wg_device, tcp_cleanup_work.work); + struct wg_tcp_socket_list_entry *entry; + bool pending; + + wg_dbg("Entering function wg_tcp_cleanup_worker\n"); + while ((entry = wg_claim_tcp_connection(wg, NULL, true))) + wg_destroy_tcp_connection_entry(wg, entry); + spin_lock_bh(&wg->tcp_connection_list_lock); + pending = !list_empty(&wg->tcp_connection_list); + spin_unlock_bh(&wg->tcp_connection_list_lock); + if (pending && READ_ONCE(wg->tcp_cleanup_scheduled)) + mod_delayed_work(system_wq, &wg->tcp_cleanup_work, + msecs_to_jiffies(WG_TCP_CLEANUP_INTERVAL_MS)); + wg_dbg("Exiting function wg_tcp_cleanup_worker\n"); +} + +struct wg_peer *wg_temp_peer_create(struct wg_device *wg) +{ + struct wg_peer *peer; + int ret = -ENOMEM; + + wg_dbg("wg_peer_create: entry with wg=%px\n", wg); + + peer = kzalloc(sizeof(struct wg_peer), GFP_KERNEL); /* BUG FIX: was kmalloc — left spinlocks, wq ptrs, flags uninitialized */ + if (unlikely(!peer)) { + wg_dbg("wg_temp_peer_create: exit with ERR_PTR(ret)\n"); + return ERR_PTR(ret); + } + + peer->device = wg; + rwlock_init(&peer->endpoint_lock); + + /* initialize TCP fields */ + peer->peer_socket = NULL; /* Initialize the peer socket to NULL */ + + peer->partial_skb = NULL; /* Initialize the partial skb pointer to NULL */ + peer->expected_len = 0; /* Initialize expected length to 0 */ + peer->received_len = 0; /* Initialize received length to 0 */ + + /* Initialize the delayed work for TCP connection retry */ + INIT_DELAYED_WORK(&peer->tcp_retry_work, wg_tcp_retry_worker); + + /* Initialize the delayed work for TCP socket removal */ + INIT_DELAYED_WORK(&peer->tcp_inbound_remove_work, wg_tcp_inbound_remove_worker); + INIT_DELAYED_WORK(&peer->tcp_outbound_remove_work, wg_tcp_outbound_remove_worker); + + /* Initialize TCP connection status flags */ + peer->tcp_established = false; + peer->tcp_pending = false; + peer->tcp_connecting = false; + peer->tcp_inbound_callbacks_set = false; + peer->tcp_outbound_callbacks_set = false; + peer->tcp_inbound_socket_data = NULL; + peer->tcp_outbound_socket_data = NULL; + peer->clean_inbound = false; + peer->clean_outbound = false; + peer->inbound_connected = false; + peer->outbound_connected = false; + peer->tcp_retry_scheduled = false; + peer->tcp_inbound_remove_scheduled = false; + peer->tcp_outbound_remove_scheduled = false; + peer->tcp_reconnect_requested = false; + peer->tcp_stopping = false; + peer->tcp_teardown_quarantined = false; + peer->tcp_outbound_remove_socket = NULL; + peer->tcp_inbound_remove_socket = NULL; + peer->tcp_roaming_connection_id = 0; + + /* Initialize the spinlock for protecting TCP-related state */ + spin_lock_init(&peer->tcp_lock); + mutex_init(&peer->tcp_socket_lock); + + /* Initialize the skb queue for the TX send queue */ + skb_queue_head_init(&peer->send_queue); + + /* Initialize the spinlock for the TX send queue */ + spin_lock_init(&peer->send_queue_lock); + + /* BUG FIX: tcp_read_lock and tcp_write_lock were never initialized — + * using uninitialized spinlocks in data_ready/write_space is UB/crash + */ + spin_lock_init(&peer->tcp_read_lock); + spin_lock_init(&peer->tcp_write_lock); + + /* Initialize the work structure, associating it with the worker functions */ + INIT_WORK(&peer->tcp_read_work, wg_tcp_read_worker); + peer->tcp_read_wq = wg->tcp_auth_wq; + + INIT_WORK(&peer->tcp_write_work, wg_tcp_write_worker); + INIT_WORK(&peer->tcp_bootstrap_work, wg_tcp_bootstrap_worker); + peer->tcp_bootstrap_socket = NULL; + INIT_WORK(&peer->tcp_promotion_work, wg_tcp_promotion_worker); + peer->tcp_promotion_connection_id = 0; + peer->tcp_promotion_worker_scheduled = false; + peer->tcp_write_wq = wg->tcp_auth_wq; + if (!peer->tcp_read_wq) { + pr_err("Provisional TCP workqueue is unavailable\n"); + goto err; + } + + /* Note this is a temp peer */ + peer->temp_peer = true; + + pr_debug("%s: Temp Peer %llu created\n", wg->dev->name, peer->internal_id); + wg_dbg("wg_temp_peer_create: exit with peer=%px\n", peer); + return peer; + +err: + kfree(peer); + wg_dbg("wg_temp_peer_create: exit with ERR_PTR(ret) on err\n"); + return ERR_PTR(ret); +} diff --git a/kernel/wg_tcp.h b/kernel/wg_tcp.h new file mode 100644 index 0000000000000000000000000000000000000000..ffc96089b8684e508a12f803209baf4c07105603 --- /dev/null +++ b/kernel/wg_tcp.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. + */ + +#ifndef _WG_TCP_H +#define _WG_TCP_H + +#include +#include + +struct endpoint; +struct net; +struct sk_buff; +struct sock; +struct socket; +struct work_struct; +struct wg_device; +struct wg_peer; +struct wg_tcp_encap_header; + +struct wg_socket_data { + struct wg_device *device; + struct wg_peer *peer; + struct socket *socket; + bool inbound; + void (*original_state_change)(struct sock *sk); + void (*original_write_space)(struct sock *sk); + void (*original_data_ready)(struct sock *sk); +}; + +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, + u8 ds); +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint); +void wg_socket_set_peer_endpoint_configured(struct wg_peer *peer, + const struct endpoint *endpoint); +void wg_socket_set_peer_endpoint_authenticated(struct wg_peer *peer, + const struct endpoint *endpoint, + u64 connection_id); +void wg_socket_set_peer_endpoint_authenticated_from_skb( + struct wg_peer *peer, const struct sk_buff *skb); +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb); +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); +void log_wireguard_endpoint(struct endpoint *endpoint); + +void wg_destruct_tcp_connection_list(struct wg_device *wg); +void print_peer_socket_info(struct wg_peer *peer); +void wg_tcp_state_change(struct sock *sk); +void wg_extract_endpoint_from_sock(struct sock *sk, struct endpoint *endpoint); +bool wg_check_potential_header_validity(struct wg_tcp_encap_header *hdr, + size_t remaining_len); + +int wg_tcp_queuepkt(struct wg_peer *peer, const void *data, size_t len); +void wg_tcp_write_space(struct sock *sk); +void wg_tcp_data_ready(struct sock *sk); +void wg_tcp_inbound_remove_worker(struct work_struct *work); +void wg_tcp_outbound_remove_worker(struct work_struct *work); +int wg_add_tcp_socket_to_list(struct wg_device *wg, struct socket *sock, + struct wg_peer *temp_peer); +void wg_remove_from_tcp_connection_list(struct wg_device *wg, + struct socket *sock); + +int wg_tcp_listener_socket_init(struct wg_device *wg, u16 port); +void wg_tcp_listener_socket_release(struct wg_device *wg); +int wg_tcp_connect(struct wg_peer *peer); +int wg_tcp_listener_worker(struct wg_device *wg, struct socket *tcp_socket); +int wg_setup_tcp_listen4(struct wg_device *wg, struct net *net, u16 port, + struct socket **listen_socket); +int wg_setup_tcp_listen6(struct wg_device *wg, struct net *net, u16 port, + struct socket **listen_socket); +int wg_tcp_listener4_thread(void *data); +int wg_tcp_listener6_thread(void *data); + +void wg_clean_peer_socket(struct wg_peer *peer, bool release, bool destroy, + bool inbound); +void wg_tcp_peer_stop(struct wg_peer *peer); +void wg_tcp_peer_request_reconnect(struct wg_peer *peer); +void wg_tcp_set_device_mark(struct wg_device *wg, u32 mark); +void wg_tcp_write_worker(struct work_struct *work); +void wg_tcp_read_worker(struct work_struct *work); +void wg_tcp_bootstrap_worker(struct work_struct *work); +void wg_tcp_promotion_worker(struct work_struct *work); +void wg_tcp_cleanup_worker(struct work_struct *work); +void wg_tcp_retry_worker(struct work_struct *work); + +#endif /* _WG_TCP_H */ diff --git a/kernel/wg_tcp_debug.c b/kernel/wg_tcp_debug.c new file mode 100644 index 0000000000000000000000000000000000000000..7c0b1da11d658f0608c5adab1e5ac929dee8d9e7 --- /dev/null +++ b/kernel/wg_tcp_debug.c @@ -0,0 +1,1207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * TCP Support Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. + */ + +#include + +#include "device.h" +#include "messages.h" +#include "peer.h" +#include "queueing.h" +#include "wg_tcp.h" +#include "wg_tcp_debug.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Enhanced SKB diagnostic and debugging function with focus on fragmentation, + * GSO/TSO, IP/TCP header analysis, PMTUD, and TCP options. + */ +void debug_skb(const struct sk_buff *askb) +{ + const struct iphdr *iph = NULL; + const struct tcphdr *tcph = NULL; + struct net_device *dev; + unsigned int frag_off_val; + bool more_frags; + int mtu = 0; + int i; + unsigned char *net_hdr; + unsigned char *end; + + if (!askb) { + printk(KERN_ERR "debug_skb: askb is NULL\n"); + return; + } + + dev = askb->dev; + net_hdr = skb_network_header(askb); + end = skb_end_pointer(askb); + + wg_dbg("==== Enhanced SKB Dump Start ====\n"); + wg_dbg("skb=%px\n", askb); + wg_dbg("head=%px, data=%px, tail=%px, end=%px\n", + askb->head, askb->data, + skb_tail_pointer(askb), end); + wg_dbg("len=%u, data_len=%u, truesize=%u\n", + askb->len, askb->data_len, askb->truesize); + wg_dbg("headroom=%u, tailroom=%u\n", + skb_headroom(askb), skb_tailroom(askb)); + wg_dbg("protocol=0x%04x, priority=%u, queue_mapping=%u\n", + ntohs(askb->protocol), askb->priority, askb->queue_mapping); + wg_dbg("pkt_type=%u, ip_summed=%u, csum_unnecessary=%u\n", + askb->pkt_type, askb->ip_summed, skb_csum_unnecessary(askb)); + + /* Device MTU */ + if (dev) { + mtu = dev->mtu; + wg_dbg("Device: %s, MTU: %d\n", dev->name, mtu); + } else { + wg_dbg("debug_skb: device not set\n"); + } + + /* GSO/TSO info */ + wg_dbg("---- Segmentation Info ----\n"); + if (skb_is_gso(askb)) { + wg_dbg("GSO enabled: size=%u, segs=%u, type=0x%x\n", + skb_shinfo(askb)->gso_size, + skb_shinfo(askb)->gso_segs, + skb_shinfo(askb)->gso_type); + if (skb_shinfo(askb)->gso_type & SKB_GSO_TCPV4) + wg_dbg(" includes TCP/IPv4\n"); + if (skb_shinfo(askb)->gso_type & SKB_GSO_TCPV6) + wg_dbg(" includes TCP/IPv6\n"); + if (skb_shinfo(askb)->gso_type & SKB_GSO_UDP) + wg_dbg(" includes UDP\n"); + } else { + wg_dbg("GSO not enabled\n"); + } + + /* Fragmentation Analysis */ + wg_dbg("---- Fragmentation Analysis ----\n"); + if (askb->data_len > 0) + { + wg_dbg("Nonlinear SKB: linear=%u, paged=%u, nr_frags=%u\n", + askb->len - askb->data_len, + askb->data_len, + skb_shinfo(askb)->nr_frags); + for (i = 0; i < skb_shinfo(askb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(askb)->frags[i]; + wg_dbg(" frag %d: size=%u, offset=%u\n", + i, skb_frag_size(frag), frag->bv_offset); + } + } else { + wg_dbg("Linear SKB: all data contiguous\n"); + } + + /* IP fragment list */ + if (skb_has_frag_list(askb)) { + struct sk_buff *iter; + int frag_count = 0; + wg_dbg("SKB fragment list present\n"); + skb_walk_frags(askb, iter) { + wg_dbg(" fraglist %d: len=%u, data_len=%u\n", + frag_count++, iter->len, iter->data_len); + } + } + + /* IP header diagnostics */ + if (net_hdr >= askb->head && net_hdr + sizeof(*iph) <= end) { + iph = ip_hdr(askb); + frag_off_val = ntohs(iph->frag_off); + more_frags = !!(frag_off_val & IP_MF); + + wg_dbg("---- IP Header ----\n"); + wg_dbg("version=%u, ihl=%u, tos=0x%x, tot_len=%u\n", + iph->version, iph->ihl, iph->tos, ntohs(iph->tot_len)); + wg_dbg("id=%u, frag_offset=%u, MF=%u\n", + ntohs(iph->id), + (frag_off_val & IP_OFFSET) << 3, + more_frags); + if (frag_off_val & IP_DF) { + wg_dbg("DF flag set\n"); + if (mtu && ntohs(iph->tot_len) > mtu) + printk(KERN_WARNING "pkt len %u > MTU %d (DF)\n", + ntohs(iph->tot_len), mtu); + } else { + wg_dbg("DF flag not set\n"); + } + wg_dbg("ttl=%u, proto=%u, src=%pI4, dst=%pI4\n", + iph->ttl, iph->protocol, + &iph->saddr, &iph->daddr); + if ((frag_off_val & IP_OFFSET) || more_frags) + wg_dbg("IP fragment (offset=%u bytes)\n", + (frag_off_val & IP_OFFSET) << 3); + else + wg_dbg("Complete IP packet\n"); + } else { + wg_dbg("Invalid or missing IP header\n"); + } + + /* TCP header diagnostics */ + if (iph && iph->protocol == IPPROTO_TCP && + skb_transport_header(askb) >= askb->head && + skb_transport_header(askb) + sizeof(*tcph) <= end) { + tcph = tcp_hdr(askb); + + wg_dbg("---- TCP Header ----\n"); + wg_dbg("sport=%u, dport=%u, seq=%u, ack=%u\n", + ntohs(tcph->source), + ntohs(tcph->dest), + ntohl(tcph->seq), + ntohl(tcph->ack_seq)); + + { + int total = ntohs(iph->tot_len); + int ip_hlen = iph->ihl * 4; + int tcp_hlen = tcph->doff * 4; + int payload = total - ip_hlen - tcp_hlen; + wg_dbg("headers: ip=%u tcp=%u payload=%u\n", + ip_hlen, tcp_hlen, payload); + } + + wg_dbg("flags=[%c%c%c%c%c%c]\n", + tcph->fin ? 'F' : '.', + tcph->syn ? 'S' : '.', + tcph->rst ? 'R' : '.', + tcph->psh ? 'P' : '.', + tcph->ack ? 'A' : '.', + tcph->urg ? 'U' : '.'); + + wg_dbg("win=%u, csum=0x%x, urg_ptr=%u\n", + ntohs(tcph->window), + ntohs(tcph->check), + ntohs(tcph->urg_ptr)); + + /* TCP options */ + if (tcph->doff > 5) { + const unsigned char *opt = (const unsigned char *)(tcph + 1); + int optlen = (tcph->doff - 5) * 4; + int j = 0; + + wg_dbg("---- TCP Options (%u bytes) ----\n", optlen); + while (j < optlen) + { + unsigned char kind = opt[j]; + + if (kind == 0) { + wg_dbg(" EOL\n"); + break; + } else if (kind == 1) { + wg_dbg(" NOP\n"); + j++; + continue; + } + + if (j + 1 >= optlen) + break; + + unsigned char length = opt[j + 1]; + if (length < 2 || j + length > optlen) + break; + + switch (kind) { + case 2: /* MSS */ + if (length == 4) + { + unsigned short mss = + ntohs(*((unsigned short *)(opt + j + 2))); + wg_dbg(" MSS=%u\n", mss); + if (mtu && mss > mtu - 40) + printk(KERN_WARNING " MSS %u > PMTU %d\n", + mss, mtu - 40); + } + break; + case 3: /* Window Scale */ + if (length == 3) + wg_dbg(" WSCALE=%u\n", opt[j + 2]); + break; + case 4: /* SACK Permitted */ + wg_dbg(" SACK_PERMITTED\n"); + break; + case 5: /* SACK Blocks */ + wg_dbg(" SACK_BLOCKS\n"); + break; + case 8: /* Timestamp */ + if (length == 10) { + u32 tsval = ntohl(*((u32 *)(opt + j + 2))); + u32 tsecr = ntohl(*((u32 *)(opt + j + 6))); + wg_dbg(" TSVAL=%u, TSecr=%u\n", + tsval, tsecr); + } + break; + default: + wg_dbg(" OPT %u LEN %u\n", kind, length); + break; + } + + j += length; + } + } + } else if (iph) { + wg_dbg("Non-TCP protocol: %u\n", iph->protocol); + } else { + wg_dbg("Invalid or missing transport header\n"); + } + wg_dbg("==== Enhanced SKB Dump End ====\n"); +} + +void debug_wireguard_packet(const unsigned char *data, size_t payload_len) +{ + size_t i, len; + + if (payload_len < sizeof(u32)) + { + wg_dbg("WireGuard packet too short\n"); + return; + } + + wg_dbg("WireGuard packet payload (%zu bytes):\n", payload_len); + for (i = 0; i < payload_len; i += 32) + { + len = min((size_t)32, payload_len - i); + wg_dbg("%*ph\n", (int)len, data + i); + } +} + +/* + * Dump the contents of an SKB—including a full raw-hex dump, + * an IP header dump, then hand off to debug_skb(), and finally + * the WireGuard payload. + */ +void debug_wireguard_skb(const struct sk_buff *skb) +{ + if (!skb) + { + printk(KERN_ERR "debug_wireguard_skb: skb is NULL\n"); + return; + } + + /* 1) Full raw buffer dump (head→end) */ + { + const unsigned char *buf = skb->head; + int buf_len = skb_end_pointer(skb) - skb->head; + int off; + + wg_dbg("==== Full raw SKB buffer dump head->end (%d bytes) ====\n", + buf_len); + for (off = 0; off < buf_len; off += 16) + { + int chunk = min(16, buf_len - off); + /* Use %*phN to get a proper hex dump of 'chunk' bytes */ + wg_dbg("%04x: %*phN\n", off, chunk, buf + off); + } + } + + /* 2) Raw IP header dump (IPv4 only) */ + { + unsigned char *net_hdr = skb_network_header(skb); + if (net_hdr >= skb->head && + net_hdr + sizeof(struct iphdr) <= skb_end_pointer(skb)) + { + const struct iphdr *iph = ip_hdr(skb); + int ihl = iph->ihl * 4; + const unsigned char *ip_ptr = net_hdr; + + wg_dbg("==== Raw IP header dump (%d bytes) ====\n", ihl); + wg_dbg("%*phN\n", ihl, ip_ptr); + } + else + { + printk(KERN_WARNING "debug_wireguard_skb: invalid IP header pointers\n"); + } + } + + /* 3) Enhanced SKB diagnostics */ + debug_skb(skb); + + /* 4) WireGuard payload dump */ + if (skb->data && skb->len > 0) { + debug_wireguard_packet(skb->data, skb->len); + } else { + wg_dbg("debug_wireguard_skb: skb data invalid (data=%p, len=%u)\n", + skb->data, skb->len); + } +} + +/* New helper function to track and debug MTU issues */ +void debug_wireguard_tcp_mtu(struct sk_buff *skb, const char *location) +{ + const struct iphdr *iph; + int actual_size, dev_mtu = 0; + + if (!skb || !location) + return; + + if (skb->dev) + dev_mtu = skb->dev->mtu; + + actual_size = skb->len; + + wg_dbg("=== WG-TCP MTU Check at %s ===\n", location); + wg_dbg("Packet size: %d bytes", actual_size); + + if (dev_mtu > 0) { + wg_dbg("Device MTU: %d bytes", dev_mtu); + if (actual_size > dev_mtu) + printk(KERN_WARNING "WARNING: Packet exceeds MTU by %d bytes\n", + actual_size - dev_mtu); + } + + if (skb_network_header_len(skb) >= sizeof(struct iphdr)) + { + iph = ip_hdr(skb); + if (iph && (ntohs(iph->frag_off) & IP_DF)) { + wg_dbg("DF flag is set - PMTUD expected to handle oversized packets\n"); + + /* Check if socket has valid route with correct PMTU */ + if (skb->sk) { + struct dst_entry *dst = skb_dst(skb); + if (dst) + { + int pmtu = dst_mtu(dst); + wg_dbg("Path MTU from route: %d bytes\n", pmtu); + if (actual_size > pmtu) + printk(KERN_WARNING "WARNING: Packet exceeds path MTU by %d bytes!\n", + actual_size - pmtu); + } else { + wg_dbg("No destination cache entry (no PMTU info)\n"); + } + } + } + } + + wg_dbg("GSO: %s (segs=%u, size=%u)\n", + skb_is_gso(skb) ? "enabled" : "disabled", + skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0, + skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : 0); + + wg_dbg("=== WG-TCP MTU Check End ===\n"); +} + +/* Helper function implementations */ +void decode_icmp_echo(const struct icmphdr *icmp_header) +{ + struct icmp_echo { + struct icmphdr hdr; + __be16 id; + __be16 sequence; + } __attribute__((packed)); + + const struct icmp_echo *echo = (const struct icmp_echo *)icmp_header; + + wg_dbg(" Identifier: %u\n", ntohs(echo->id)); + wg_dbg(" Sequence Number: %u\n", ntohs(echo->sequence)); +} + +void decode_icmp_dest_unreachable(const struct icmphdr *icmp_header) +{ + wg_dbg(" Gateway Address: %pI4\n", &icmp_header->un.gateway); + + switch (icmp_header->code) { + case ICMP_NET_UNREACH: + wg_dbg(" Code: Network Unreachable\n"); + break; + case ICMP_HOST_UNREACH: + wg_dbg(" Code: Host Unreachable\n"); + break; + case ICMP_PROT_UNREACH: + wg_dbg(" Code: Protocol Unreachable\n"); + break; + case ICMP_PORT_UNREACH: + wg_dbg(" Code: Port Unreachable\n"); + break; + /* Add more cases as needed */ + default: + wg_dbg(" Code: %u\n", icmp_header->code); + break; + } +} + +void decode_icmp_time_exceeded(const struct icmphdr *icmp_header) +{ + wg_dbg(" Unused Field: %u\n", ntohl(icmp_header->un.gateway)); + + switch (icmp_header->code) { + case ICMP_EXC_TTL: + wg_dbg(" Code: Time To Live Exceeded\n"); + break; + case ICMP_EXC_FRAGTIME: + wg_dbg(" Code: Fragment Reassembly Time Exceeded\n"); + break; + default: + wg_dbg(" Code: %u\n", icmp_header->code); + break; + } +} + +void decode_icmp_other(const struct icmphdr *icmp_header) +{ + wg_dbg(" Rest of Header (Raw Data): %u\n", ntohl(icmp_header->un.gateway)); +} + +/* + * Function to decode and print TCP, UDP, and ICMP parameters + * Now accepts 'const char *prefix' and conditionally linearizes fragmented packets + */ +void decode_and_print_packet(const struct sk_buff *skb, const char *prefix) +{ + struct iphdr *ip_header; + struct tcphdr *tcp_header; + struct udphdr *udp_header; + struct icmphdr *icmp_header; + unsigned int ip_header_length; + unsigned int tcp_header_length; + + /* Retrieve the IP header using helper function */ + ip_header = ip_hdr(skb); + + /* Ensure the skb contains enough data for IP header */ + if (skb->len < sizeof(struct iphdr)) { + wg_dbg("%sPacket too short for IP header\n", prefix); + return; + } + + ip_header_length = ip_header->ihl * 4; + + /* Verify that the IP header length is valid */ + if (ip_header_length < sizeof(struct iphdr)) { + wg_dbg("%sInvalid IP header length: %u bytes\n", prefix, ip_header_length); + return; + } + + /* Ensure the skb has the complete IP header */ + if (skb->len < ip_header_length) { + wg_dbg("%sIncomplete IP header in skb\n", prefix); + return; + } + + /* + * Check if the packet is fragmented + * ip_header->frag_off is in network byte order; convert to host byte order + */ + if (ntohs(ip_header->frag_off) & (IP_MF | IP_OFFSET)) { + /* Packet is fragmented; attempt to linearize */ + if (skb_linearize((struct sk_buff *)skb) < 0) { + wg_dbg("%sFailed to linearize skb for fragmented packet\n", prefix); + return; + } + + /* After linearization, re-fetch the IP header as skb data may have changed */ + ip_header = ip_hdr(skb); + ip_header_length = ip_header->ihl * 4; + + /* Re-validate IP header after linearization */ + if (ip_header_length < sizeof(struct iphdr)) { + wg_dbg("%sInvalid IP header length after linearization: %u bytes\n", prefix, ip_header_length); + return; + } + + if (skb->len < ip_header_length) { + wg_dbg("%sIncomplete IP header in skb after linearization\n", prefix); + return; + } + } + + /* Determine the protocol and handle accordingly */ + switch (ip_header->protocol) { + case IPPROTO_TCP: + /* Ensure the skb has enough data for the TCP header */ + if (skb->len < ip_header_length + sizeof(struct tcphdr)) { + wg_dbg("%sPacket too short for TCP header\n", prefix); + return; + } + + /* Retrieve the TCP header using helper function */ + tcp_header = tcp_hdr(skb); + if (!tcp_header) { + wg_dbg("%sFailed to retrieve TCP header\n", prefix); + return; + } + + tcp_header_length = tcp_header->doff * 4; + + /* Validate TCP header length */ + if (tcp_header_length < sizeof(struct tcphdr)) { + wg_dbg("%sInvalid TCP header length: %u bytes\n", prefix, tcp_header_length); + return; + } + + /* Ensure the skb has the complete TCP header */ + if (skb->len < ip_header_length + tcp_header_length) { + wg_dbg("%sPacket too short for complete TCP header\n", prefix); + return; + } + + /* Define a buffer to hold the TCP flags string */ + char tcp_flags[64]; + tcp_flags[0] = '\0'; /* Initialize the string */ + + /* Append each TCP flag if it is set */ + if (tcp_header->fin) + strlcat(tcp_flags, "FIN ", sizeof(tcp_flags)); + if (tcp_header->syn) + strlcat(tcp_flags, "SYN ", sizeof(tcp_flags)); + if (tcp_header->rst) + strlcat(tcp_flags, "RST ", sizeof(tcp_flags)); + if (tcp_header->psh) + strlcat(tcp_flags, "PSH ", sizeof(tcp_flags)); + if (tcp_header->ack) + strlcat(tcp_flags, "ACK ", sizeof(tcp_flags)); + if (tcp_header->urg) + strlcat(tcp_flags, "URG ", sizeof(tcp_flags)); + if (tcp_header->ece) + strlcat(tcp_flags, "ECE ", sizeof(tcp_flags)); + if (tcp_header->cwr) + strlcat(tcp_flags, "CWR ", sizeof(tcp_flags)); + + /* Print TCP parameters with prefix */ + wg_dbg("%s#### TCP Packet: " + "S: %pI4 " + "D: %pI4 " + "SP: %u " + "DP: %u " + "SN: %u " + "AN: %u " + "DO: %u bytes " + "F: %s " + "WS: %u " + "C: 0x%04x " + "U: %u " + "skb: %px len: %u\n", + prefix, + &ip_header->saddr, + &ip_header->daddr, + ntohs(tcp_header->source), + ntohs(tcp_header->dest), + ntohl(tcp_header->seq), + ntohl(tcp_header->ack_seq), + tcp_header_length, + tcp_flags, + ntohs(tcp_header->window), + ntohs(tcp_header->check), + ntohs(tcp_header->urg_ptr), + skb, skb->len); + break; + + case IPPROTO_UDP: + /* Ensure the skb has enough data for the UDP header */ + if (skb->len < ip_header_length + sizeof(struct udphdr)) { + wg_dbg("%sPacket too short for UDP header\n", prefix); + return; + } + + /* Retrieve the UDP header using helper function */ + udp_header = udp_hdr(skb); + if (!udp_header) { + wg_dbg("%sFailed to retrieve UDP header\n", prefix); + return; + } + + /* Print UDP parameters with prefix */ + wg_dbg("%s#### UDP Packet: " + "S: %pI4 " + "D: %pI4 " + "SP: %u " + "DP: %u " + "L: %u " + "C: 0x%04x " + "skb: %px len: %u\n", + prefix, + &ip_header->saddr, + &ip_header->daddr, + ntohs(udp_header->source), + ntohs(udp_header->dest), + ntohs(udp_header->len), + ntohs(udp_header->check), + skb, skb->len); + + /* Print skb address and length */ + wg_dbg("%sskb address: %px, skb length: %u\n", prefix, skb, skb->len); + break; + + case IPPROTO_ICMP: + /* Ensure the skb has enough data for the ICMP header */ + if (skb->len < ip_header_length + sizeof(struct icmphdr)) { + wg_dbg("%sPacket too short for ICMP header\n", prefix); + return; + } + + /* Retrieve the ICMP header using helper function */ + icmp_header = icmp_hdr(skb); + if (!icmp_header) { + wg_dbg("%sFailed to retrieve ICMP header\n", prefix); + return; + } + + /* Print basic ICMP parameters with prefix */ + wg_dbg("%s#### ICMP Packet: " + "S: %pI4 " + "D: %pI4 " + "Type: %u " + "Code: %u " + "C: 0x%04x " + "skb: %px len: %u\n", + prefix, + &ip_header->saddr, + &ip_header->daddr, + icmp_header->type, + icmp_header->code, + ntohs(icmp_header->checksum), + skb, skb->len); + + /* Decode the "Rest of the Header" based on Type */ + switch (icmp_header->type) { + case ICMP_ECHO: + case ICMP_ECHOREPLY: + decode_icmp_echo(icmp_header); + break; + + case ICMP_DEST_UNREACH: + decode_icmp_dest_unreachable(icmp_header); + break; + + case ICMP_TIME_EXCEEDED: + decode_icmp_time_exceeded(icmp_header); + break; + + default: + decode_icmp_other(icmp_header); + break; + } + + /* Print skb address and length */ + wg_dbg("%sskb address: %px, skb length: %u\n", prefix, skb, skb->len); + break; + + default: + /* Handle unsupported protocols */ + /* BUG FIX: format string was split by comma after D: %pI4\n — + * "skb: %px len: %u\n" was passed as %s arg, shifting all args (UB/crash) + */ + wg_dbg("%s#### Unsupported Protocol: %u " + "S: %pI4 " + "D: %pI4 " + "skb: %px len: %u\n", + prefix, + ip_header->protocol, + &ip_header->saddr, + &ip_header->daddr, + skb, skb->len); + + /* Print skb address and length */ + wg_dbg("%sskb address: %px, skb length: %u\n", prefix, skb, skb->len); + break; + } +} + +void print_skbuff_head_info(const char *label, struct sk_buff_head *queue) +{ + const struct sk_buff *skb; + unsigned long flags; + + wg_dbg("%s:\n", label); + if (!queue) { + wg_dbg("Queue is NULL\n"); + return; + } + + spin_lock_irqsave(&queue->lock, flags); + skb_queue_walk(queue, skb) { + wg_dbg("Packet: len=%u, data_len=%u, users=%d\n", + skb->len, skb->data_len, refcount_read(&skb->users)); + } + spin_unlock_irqrestore(&queue->lock, flags); +} + +void print_wg_peer(struct wg_peer *peer) +{ + if (!peer || IS_ERR(peer)) { + printk(KERN_ERR "NULL wg_peer provided\n"); + return; + } + + wg_dbg("WG Peer Complete Diagnostic Info:\n"); + wg_dbg("Device Pointer: %px, Serial Work CPU: %d, " + "Is Dead: %d, (Device) Transport Mode: %u\n", + peer->device, peer->serial_work_cpu, peer->is_dead, + peer->device->transport); + wg_dbg("RX Bytes: %llu, TX Bytes: %llu, Internal ID: %llu\n", + peer->rx_bytes, peer->tx_bytes, peer->internal_id); + wg_dbg("Last Sent Handshake: %llu\n", + atomic64_read(&peer->last_sent_handshake)); + + /* Endpoint info */ + wg_dbg("Endpoint Address Family: %u\n", + peer->endpoint.addr.sa_family); + if (peer->endpoint.addr.sa_family == AF_INET) { + wg_dbg("IPv4 Address: %pI4, IPv4 Source: %pI4, " + "Interface: %d\n", + &peer->endpoint.addr4.sin_addr, &peer->endpoint.src4, + peer->endpoint.src_if4); + } else if (peer->endpoint.addr.sa_family == AF_INET6) { + wg_dbg("IPv6 Address: %pI6c, IPv6 Source: %pI6c\n", + &peer->endpoint.addr6.sin6_addr, &peer->endpoint.src6); + } + + /* Correctly accessing sk_buff_head queues */ + if (!skb_queue_empty(&peer->staged_packet_queue)) { + print_skbuff_head_info("Staged Packet Queue", + &peer->staged_packet_queue); + } else { + wg_dbg("Staged Packet Queue: NULL\n"); + } + + /* Additional diagnostics and corrections for TCP */ + if (peer->peer_socket) { + wg_dbg("TCP Socket: %px, Established: %d\n", + peer->peer_socket, peer->tcp_established); + if (!skb_queue_empty(&peer->send_queue)) { + print_skbuff_head_info("TCP Packet Queue", + &peer->send_queue); + } else { + wg_dbg("TCP Packet Queue: NULL\n"); + } + } else { + wg_dbg("TCP Socket: NULL\n"); + } + + /* Timer diagnostics */ + wg_dbg("Timer for Retransmit Handshake Expires: %ld\n", + peer->timer_retransmit_handshake.expires); + wg_dbg("Timer for Sending Keepalive Expires: %ld\n", + peer->timer_send_keepalive.expires); + wg_dbg("Timer for New Handshake Expires: %ld\n", + peer->timer_new_handshake.expires); + wg_dbg("Timer for Zero Key Material Expires: %ld\n", + peer->timer_zero_key_material.expires); + wg_dbg("Timer for Persistent Keepalive Expires: %ld\n", + peer->timer_persistent_keepalive.expires); + + /* RCU and reference count */ + wg_dbg("RCU Head Address: %px, Reference Count: %d\n", + &peer->rcu, kref_read(&peer->refcount)); +} + +void print_crypt_queue(const char *label, struct crypt_queue *queue) +{ + if (!queue) { + wg_dbg("%s: NULL\n", label); + return; + } + + wg_dbg("%s:\n", label); + wg_dbg(" Last CPU used: %d\n", queue->last_cpu); + if (queue->worker) + wg_dbg(" Worker pointer: %px\n", queue->worker); + else + wg_dbg(" Worker: NULL\n"); +} + +void print_wg_device(struct wg_device *device) +{ + if (!device) { + printk(KERN_ERR "NULL wg_device provided\n"); + return; + } + + wg_dbg("WG Device Diagnostic Info:\n"); + + if (device->dev) + wg_dbg("Net device: %s\n", device->dev->name); + else + wg_dbg("Net device: NULL\n"); + + print_crypt_queue("Encrypt Queue", &(device->encrypt_queue)); + print_crypt_queue("Decrypt Queue", &(device->decrypt_queue)); + print_crypt_queue("Handshake Queue", &(device->handshake_queue)); + + if (rcu_access_pointer(device->tcp_listen_socket4)) + wg_dbg("IPv4 Socket: %px\n", device->tcp_listen_socket4); + else + wg_dbg("IPv4 Socket: NULL\n"); + + if (rcu_access_pointer(device->tcp_listen_socket6)) + wg_dbg("IPv6 Socket: %px\n", device->tcp_listen_socket6); + else + wg_dbg("IPv6 Socket: NULL\n"); + + if (rcu_access_pointer(device->tcp_listen_socket4)) + wg_dbg("TCP Listener IPv4 Socket: %px\n", + device->tcp_listen_socket4); + else + wg_dbg("TCP Listener IPv4 Socket: NULL\n"); + + if (rcu_access_pointer(device->tcp_listen_socket6)) + wg_dbg("TCP Listener IPv6 Socket: %px\n", + device->tcp_listen_socket6); + else + wg_dbg("TCP Listener IPv6 Socket: NULL\n"); + + if (device->creating_net) + wg_dbg("Creating net namespace: %px\n", + device->creating_net); + else + wg_dbg("Creating net namespace: NULL\n"); + + wg_dbg("Static Identity: (printing details not implemented)\n"); + wg_dbg("Workqueues and other components would similarly have their details printed based on available data.\n"); + + wg_dbg("FW Mark: %u, Incoming Port: %u, Transport: %u\n", device->fwmark, device->incoming_port, device->transport); + wg_dbg("Handshake queue length: %d\n", atomic_read(&device->handshake_queue_len)); + wg_dbg("Number of Peers: %u, Device Update Generation: %u\n", device->num_peers, device->device_update_gen); +} + +void print_tcp_socket_info(struct socket *sock, const char *label) { + struct sock *sk; + struct wg_socket_data *user_data; + int tcp_state = -1; + + if (sock && sock->sk) { + sk = sock->sk; + user_data = (struct wg_socket_data *)sk->sk_user_data; + tcp_state = (sk->sk_protocol == IPPROTO_TCP) ? sk->sk_state : -1; + if (user_data) { + wg_dbg("%s: socket=%px, sk_user_data=%px (device=%px, peer=%px, inbound=%d), TCP state=%d\n", + label, sock, user_data, user_data->device, user_data->peer, user_data->inbound, tcp_state); + } else { + wg_dbg("%s: socket=%px, sk_user_data=NULL, TCP state=%d\n", + label, sock, tcp_state); + } + } else { + wg_dbg("%s: Socket or sk is NULL\n", label); + } +} + +/* Function to print compact diagnostic information for all sockets in a peer */ +void print_peer_socket_info(struct wg_peer *peer) { + if (!peer) { + wg_dbg("print_peer_socket_info: peer is NULL\n"); + return; + } + + /* Print the pointers to the main sockets in the peer */ + wg_dbg("Peer: %px, peer_socket=%px, inbound_socket=%px, outbound_socket=%px\n", + peer, peer->peer_socket, peer->inbound_socket, peer->outbound_socket); + + /* Print inbound timestamp */ + wg_dbg("Inbound timestamp: %llu ns\n", ktime_to_ns(peer->inbound_timestamp)); + + /* Print outbound timestamp */ + wg_dbg("Outbound timestamp: %llu ns\n", ktime_to_ns(peer->outbound_timestamp)); + + /* Print combined information for inbound socket */ + if (peer->inbound_socket) { + print_tcp_socket_info(peer->inbound_socket, "Inbound Socket"); + } else { + wg_dbg("Inbound Socket is NULL\n"); + } + + /* Print combined information for outbound socket */ + if (peer->outbound_socket) { + print_tcp_socket_info(peer->outbound_socket, "Outbound Socket"); + } else { + wg_dbg("Outbound Socket is NULL\n"); + } + + /* Additional validation check */ + if (peer->peer_socket == peer->inbound_socket) { + wg_dbg("peer_socket matches inbound_socket\n"); + } else if (peer->peer_socket == peer->outbound_socket) { + wg_dbg("peer_socket matches outbound_socket\n"); + } else { + printk(KERN_WARNING "peer_socket does not match inbound_socket or outbound_socket\n"); + } +} + +/* ============================================================================ + * WireGuard-over-TCP Diagnostic Framework + * + * Comprehensive printk diagnostics for troubleshooting TCP-mode inefficiencies: + * - Excessive loss/retransmits + * - Window/cwnd issues + * - Short writes + * - Receive-side head-of-line stalls + * + * View logs with: dmesg | grep "wg-tcp-diag\|tcpdiag" + * NOTE: Rate limiting disabled for complete diagnostics + * ============================================================================ + */ + +/* Aggregate statistics counters */ +atomic64_t wg_tcp_stats_tx_bytes = ATOMIC64_INIT(0); +atomic64_t wg_tcp_stats_rx_bytes = ATOMIC64_INIT(0); +atomic64_t wg_tcp_stats_tx_packets = ATOMIC64_INIT(0); +atomic64_t wg_tcp_stats_rx_packets = ATOMIC64_INIT(0); +atomic64_t wg_tcp_stats_tx_eagain = ATOMIC64_INIT(0); +atomic64_t wg_tcp_stats_tx_errors = ATOMIC64_INIT(0); +atomic64_t wg_tcp_stats_rx_errors = ATOMIC64_INIT(0); +atomic64_t wg_tcp_stats_short_writes = ATOMIC64_INIT(0); +/* Note: retransmits counter shows tp->total_retrans from dump_sock, not incremented here */ +atomic64_t wg_tcp_stats_retransmits = ATOMIC64_INIT(0); + +/* Portable congestion window accessor */ +static inline u32 wg_tcp_get_cwnd(const struct tcp_sock *tp) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,0,0) + return tcp_snd_cwnd(tp); +#else + return tp->snd_cwnd; +#endif +} + +/* TCP state name lookup */ +static const char *wg_tcp_diag_state_name(u8 state) +{ + switch (state) { + case TCP_ESTABLISHED: return "ESTABLISHED"; + case TCP_SYN_SENT: return "SYN_SENT"; + case TCP_SYN_RECV: return "SYN_RECV"; + case TCP_FIN_WAIT1: return "FIN_WAIT1"; + case TCP_FIN_WAIT2: return "FIN_WAIT2"; + case TCP_TIME_WAIT: return "TIME_WAIT"; + case TCP_CLOSE: return "CLOSE"; + case TCP_CLOSE_WAIT: return "CLOSE_WAIT"; + case TCP_LAST_ACK: return "LAST_ACK"; + case TCP_LISTEN: return "LISTEN"; + case TCP_CLOSING: return "CLOSING"; + case TCP_NEW_SYN_RECV:return "NEW_SYN_RECV"; + default: return "UNKNOWN"; + } +} + +/* Format endpoint addresses for logging */ +static void wg_tcp_diag_format_endpoints(struct sock *sk, + char *lbuf, size_t lbuf_len, + char *rbuf, size_t rbuf_len) +{ + struct inet_sock *inet; + + if (!sk) { + snprintf(lbuf, lbuf_len, "sk=null"); + snprintf(rbuf, rbuf_len, "sk=null"); + return; + } + + inet = inet_sk(sk); + + if (sk->sk_family == AF_INET) { + snprintf(lbuf, lbuf_len, "%pI4:%u", + &inet->inet_rcv_saddr, ntohs(inet->inet_sport)); + snprintf(rbuf, rbuf_len, "%pI4:%u", + &inet->inet_daddr, ntohs(inet->inet_dport)); + return; + } +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) { + snprintf(lbuf, lbuf_len, "[%pI6c]:%u", + &sk->sk_v6_rcv_saddr, ntohs(inet->inet_sport)); + snprintf(rbuf, rbuf_len, "[%pI6c]:%u", + &sk->sk_v6_daddr, ntohs(inet->inet_dport)); + return; + } +#endif + snprintf(lbuf, lbuf_len, "fam=%u", sk->sk_family); + snprintf(rbuf, rbuf_len, "fam=%u", sk->sk_family); +} + +/* Peek at WireGuard message type from skb */ +static u32 wg_tcp_diag_peek_msg_type(const struct sk_buff *skb) +{ + const struct message_header *h; + + if (!skb || skb->len < sizeof(*h)) + return 0; + + h = (const struct message_header *)skb->data; + return le32_to_cpu(h->type); +} + +/* Comprehensive socket dump - includes all TCP metrics */ +void wg_tcp_diag_dump_sock(struct sock *sk, const char *where, + ssize_t io_bytes, size_t io_wanted) +{ + struct wg_socket_data *sd; + struct wg_peer *peer = NULL; + struct wg_device *wg = NULL; + bool inbound = false; + const char *devname = "wireguard"; + u64 peer_id = 0; + char laddr[80], raddr[80]; + struct tcp_sock *tp; + struct inet_connection_sock *icsk; + u32 srtt_us, rto_ms, cwnd; + u32 wmem, rmem; + u32 writeq_len, recvq_len; + + if (!sk || IS_ERR(sk)) + return; + if (sk->sk_protocol != IPPROTO_TCP) + return; + + sd = READ_ONCE(sk->sk_user_data); + if (sd && !IS_ERR(sd)) { + peer = sd->peer; + wg = sd->device; + inbound = sd->inbound; + if (wg && wg->dev) + devname = wg->dev->name; + if (peer && !IS_ERR(peer)) + peer_id = peer->internal_id; + } + + wg_tcp_diag_format_endpoints(sk, laddr, sizeof(laddr), raddr, sizeof(raddr)); + + tp = tcp_sk(sk); + icsk = inet_csk(sk); + cwnd = wg_tcp_get_cwnd(tp); + + /* tp->srtt_us is scaled by 8 (<< 3) */ + srtt_us = tp->srtt_us >> 3; + rto_ms = jiffies_to_msecs(icsk->icsk_rto); + + wmem = sk_wmem_alloc_get(sk); + rmem = sk_rmem_alloc_get(sk); + writeq_len = skb_queue_len(&sk->sk_write_queue); + recvq_len = skb_queue_len(&sk->sk_receive_queue); + + wg_diag( + "%s: tcpdiag[%s] peer=%llu inbound=%d sk=%px state=%s(%u) err=%d shut=%u io=%zd/%zu " + "lcl=%s rmt=%s " + "snd_wnd=%u rcv_wnd=%u cwnd=%u ssthresh=%u " + "snd_una=%u snd_nxt=%u rcv_nxt=%u inflight=%u " + "sndbuf=%u rcvbuf=%u wmem=%u rmem=%u wmemq=%u " + "writeq=%u recvq=%u " + "mss=%u advmss=%u wscale(snd=%u rcv=%u) nonagle=%u " + "rto=%ums srtt=%uus rttvar=%uus " + "pkts_out=%u retrans_out=%u lost_out=%u sacked_out=%u total_retrans=%u " + "segs_in=%u segs_out=%u bytes_sent=%llu bytes_acked=%llu bytes_received=%llu cc=%s ca_state=%u\n", + devname, where ? where : "?", + peer_id, inbound, sk, + wg_tcp_diag_state_name(sk->sk_state), sk->sk_state, + sk->sk_err, sk->sk_shutdown, + io_bytes, io_wanted, + laddr, raddr, + tp->snd_wnd, tp->rcv_wnd, cwnd, tp->snd_ssthresh, + tp->snd_una, tp->snd_nxt, tp->rcv_nxt, tp->snd_nxt - tp->snd_una, + sk->sk_sndbuf, sk->sk_rcvbuf, + wmem, rmem, sk->sk_wmem_queued, + writeq_len, recvq_len, + tp->mss_cache, tp->advmss, + tp->rx_opt.snd_wscale, tp->rx_opt.rcv_wscale, tp->nonagle, + rto_ms, srtt_us, tp->rttvar_us, + tp->packets_out, tp->retrans_out, tp->lost_out, tp->sacked_out, + tp->total_retrans, + tp->segs_in, tp->segs_out, + (unsigned long long)tp->bytes_sent, + (unsigned long long)tp->bytes_acked, + (unsigned long long)tp->bytes_received, + icsk->icsk_ca_ops ? icsk->icsk_ca_ops->name : "?", + icsk->icsk_ca_state); +} + +/* Check and log TCP pressure indicators */ +void wg_tcp_diag_pressure(struct sock *sk, u64 peer_id) +{ + struct tcp_sock *tp; + struct inet_connection_sock *icsk; + u32 cwnd; + bool pressure = false; + char reasons[128] = ""; + int pos = 0; + + if (!sk) + return; + + tp = tcp_sk(sk); + icsk = inet_csk(sk); + cwnd = wg_tcp_get_cwnd(tp); + + if (tp->snd_wnd < tp->mss_cache * 2) { + pressure = true; + pos += snprintf(reasons + pos, sizeof(reasons) - pos, "small_wnd "); + } + if (cwnd < 4) { + pressure = true; + pos += snprintf(reasons + pos, sizeof(reasons) - pos, "cwnd_low "); + } + if (tp->retrans_out > 0) { + pressure = true; + pos += snprintf(reasons + pos, sizeof(reasons) - pos, "retrans "); + } + if (tp->lost_out > 0) { + pressure = true; + pos += snprintf(reasons + pos, sizeof(reasons) - pos, "lost "); + } + if (sk->sk_wmem_queued > (sk->sk_sndbuf * 4 / 5)) { + pressure = true; + pos += snprintf(reasons + pos, sizeof(reasons) - pos, "wmem_full "); + } + if (tp->snd_wnd == 0) { + pressure = true; + pos += snprintf(reasons + pos, sizeof(reasons) - pos, "ZERO_WND "); + } + + if (pressure) { + printk(KERN_WARNING + "wg-tcp-diag [PRESSURE] peer=%llu: %s| " + "snd_wnd=%u cwnd=%u ssthresh=%u mss=%u | " + "retrans=%u lost=%u rto=%ums | " + "wmem=%d/%d\n", + peer_id, reasons, + tp->snd_wnd, cwnd, tp->snd_ssthresh, tp->mss_cache, + tp->retrans_out, tp->lost_out, jiffies_to_msecs(icsk->icsk_rto), + sk->sk_wmem_queued, sk->sk_sndbuf); + } +} + +/* Log aggregate statistics */ +void wg_tcp_diag_aggregate(void) +{ + wg_diag("wg-tcp-diag [STATS]: " + "tx=%lld bytes/%lld pkts rx=%lld bytes/%lld pkts | " + "eagain=%lld short=%lld tx_err=%lld rx_err=%lld retrans=%lld\n", + atomic64_read(&wg_tcp_stats_tx_bytes), + atomic64_read(&wg_tcp_stats_tx_packets), + atomic64_read(&wg_tcp_stats_rx_bytes), + atomic64_read(&wg_tcp_stats_rx_packets), + atomic64_read(&wg_tcp_stats_tx_eagain), + atomic64_read(&wg_tcp_stats_short_writes), + atomic64_read(&wg_tcp_stats_tx_errors), + atomic64_read(&wg_tcp_stats_rx_errors), + atomic64_read(&wg_tcp_stats_retransmits)); +} + +/* ============================================================================ + * End of TCP Diagnostic Framework + * ============================================================================ + */ diff --git a/kernel/wg_tcp_debug.h b/kernel/wg_tcp_debug.h new file mode 100644 index 0000000000000000000000000000000000000000..82a32ed5fa24ad1fb97ef2a92e428233e620c8b1 --- /dev/null +++ b/kernel/wg_tcp_debug.h @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024-2026 Jeff Nathan and Dragos Ruiu. All Rights Reserved. + */ +/* + * WireGuard TCP Debug Macros + * + * Two independent levels, enabled via compiler flags: + * + * -DWG_TCP_VERBOSE Very verbose: function enter/exit traces, parameter + * dumps, packet header parsing. Extremely noisy. + * + * -DWG_TCP_DIAG TCP performance diagnostics: per-packet socket state + * dumps (cwnd, rtt, retrans, etc). Useful for debugging + * throughput and congestion issues. + * + * Build examples (add to Makefile ccflags-y or command line): + * make ... EXTRA_CFLAGS="-DWG_TCP_VERBOSE -DWG_TCP_DIAG" # everything + * make ... EXTRA_CFLAGS="-DWG_TCP_DIAG" # perf diag only + * make ... # no debug + * + * Error messages (KERN_ERR / pr_err) are always compiled in. + */ + +#ifndef _WG_TCP_DEBUG_H +#define _WG_TCP_DEBUG_H + +#include +#include +#include + +#ifdef WG_TCP_VERBOSE +#define wg_dbg(fmt, ...) printk(KERN_INFO fmt, ##__VA_ARGS__) +#else +#define wg_dbg(fmt, ...) do {} while (0) +#endif + +#ifdef WG_TCP_DIAG +#define wg_diag(fmt, ...) pr_info(fmt, ##__VA_ARGS__) +#define WG_TCP_DIAG_ENABLED 1 +#else +#define wg_diag(fmt, ...) do {} while (0) +#define WG_TCP_DIAG_ENABLED 0 +#endif + +struct sk_buff; +struct crypt_queue; +struct sk_buff_head; +struct sock; +struct socket; +struct wg_device; +struct wg_peer; + +void debug_skb(const struct sk_buff *askb); +void debug_wireguard_packet(const unsigned char *data, + size_t payload_len); +void debug_wireguard_skb(const struct sk_buff *skb); +void debug_wireguard_tcp_mtu(struct sk_buff *skb, const char *location); + +void decode_icmp_echo(const struct icmphdr *icmp_header); +void decode_icmp_dest_unreachable(const struct icmphdr *icmp_header); +void decode_icmp_time_exceeded(const struct icmphdr *icmp_header); +void decode_icmp_other(const struct icmphdr *icmp_header); +void decode_and_print_packet(const struct sk_buff *skb, const char *prefix); + +void print_wg_peer(struct wg_peer *peer); +void print_crypt_queue(const char *label, struct crypt_queue *queue); +void print_wg_device(struct wg_device *device); +void print_skbuff_head_info(const char *label, struct sk_buff_head *queue); +void print_tcp_socket_info(struct socket *sock, const char *label); +void print_peer_socket_info(struct wg_peer *peer); + +#ifdef WG_TCP_DIAG +extern atomic64_t wg_tcp_stats_tx_bytes; +extern atomic64_t wg_tcp_stats_rx_bytes; +extern atomic64_t wg_tcp_stats_tx_packets; +extern atomic64_t wg_tcp_stats_rx_packets; +extern atomic64_t wg_tcp_stats_tx_eagain; +extern atomic64_t wg_tcp_stats_tx_errors; +extern atomic64_t wg_tcp_stats_rx_errors; +extern atomic64_t wg_tcp_stats_short_writes; +extern atomic64_t wg_tcp_stats_retransmits; + +void wg_tcp_diag_dump_sock(struct sock *sk, const char *where, + int result, unsigned int queued); +void wg_tcp_diag_pressure(struct sock *sk, u64 peer_id); +void wg_tcp_diag_aggregate(void); +#endif + +#endif /* _WG_TCP_DEBUG_H */ diff --git a/kernel/wireguard_tcp_uapi.h b/kernel/wireguard_tcp_uapi.h new file mode 100644 index 0000000000000000000000000000000000000000..72bafb4772dae1caac64ad31ec973646230a41a8 --- /dev/null +++ b/kernel/wireguard_tcp_uapi.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _WG_TCP_UAPI_H +#define _WG_TCP_UAPI_H + +#include "../include/uapi/linux/wireguard.h" + +#endif diff --git a/tools/Makefile b/tools/Makefile index c37865bfa5b301ebdb6089bee6c73eb41b1b3eb2..5b68b32d3111ce04ea653599cfb5a1c0ddac156f 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -39,7 +39,7 @@ PLATFORM ?= $(shell uname -s | tr '[:upper:]' '[:lower:]') CFLAGS ?= -O3 ifneq ($(wildcard uapi/$(PLATFORM)/.),) -CFLAGS += -isystem uapi/$(PLATFORM) +CFLAGS += -Iuapi/$(PLATFORM) endif CFLAGS += -std=gnu99 -D_GNU_SOURCE CFLAGS += -Wall -Wextra diff --git a/tools/config.c b/tools/config.c index 0e145d4cd76c71487b51043bf5898523d390739c..866035cefd762723427ca68dfd6c6d8e254eddf0 100644 --- a/tools/config.c +++ b/tools/config.c @@ -305,6 +305,19 @@ err: return false; } +static inline bool parse_transport(uint8_t *transport, const char *value) +{ + if (!strcasecmp(value, "udp")) + *transport = WG_TRANSPORT_UDP; + else if (!strcasecmp(value, "tcp")) + *transport = WG_TRANSPORT_TCP; + else { + fprintf(stderr, "Transport is neither udp nor tcp: `%s'\n", value); + return false; + } + return true; +} + static bool validate_netmask(struct wgallowedip *allowedip) { uint32_t *ip; @@ -473,6 +486,10 @@ static bool process_line(struct config_ctx *ctx, const char *line) ret = parse_key(ctx->device->private_key, value); if (ret) ctx->device->flags |= WGDEVICE_HAS_PRIVATE_KEY; + } else if (key_match("Transport") || key_match("TransportMode")) { + ret = parse_transport(&ctx->device->transport, value); + if (ret) + ctx->device->flags |= WGDEVICE_HAS_TRANSPORT; } else goto error; } else if (ctx->is_peer_section) { @@ -612,6 +629,12 @@ struct wgdevice *config_read_cmd(const char *argv[], int argc) device->flags |= WGDEVICE_HAS_PRIVATE_KEY; argv += 2; argc -= 2; + } else if (!strcmp(argv[0], "transport") && argc >= 2 && !peer) { + if (!parse_transport(&device->transport, argv[1])) + goto error; + device->flags |= WGDEVICE_HAS_TRANSPORT; + argv += 2; + argc -= 2; } else if (!strcmp(argv[0], "peer") && argc >= 2) { struct wgpeer *new_peer = calloc(1, sizeof(*new_peer)); diff --git a/tools/containers.h b/tools/containers.h index 30a673918517feeb28dae64eeec37c5d5ae0ee55..140283f0b02f32e462f4f034486e2700425f850f 100644 --- a/tools/containers.h +++ b/tools/containers.h @@ -22,6 +22,11 @@ #define WG_KEY_LEN 32 #endif +#ifndef WG_TRANSPORT_UDP +#define WG_TRANSPORT_UDP 0 +#define WG_TRANSPORT_TCP 1 +#endif + /* Cross platform __kernel_timespec */ struct timespec64 { int64_t tv_sec; @@ -76,7 +81,8 @@ enum { WGDEVICE_HAS_PRIVATE_KEY = 1U << 1, WGDEVICE_HAS_PUBLIC_KEY = 1U << 2, WGDEVICE_HAS_LISTEN_PORT = 1U << 3, - WGDEVICE_HAS_FWMARK = 1U << 4 + WGDEVICE_HAS_FWMARK = 1U << 4, + WGDEVICE_HAS_TRANSPORT = 1U << 5 }; struct wgdevice { @@ -90,6 +96,7 @@ struct wgdevice { uint32_t fwmark; uint16_t listen_port; + uint8_t transport; struct wgpeer *first_peer, *last_peer; }; diff --git a/tools/ipc-linux.h b/tools/ipc-linux.h index 9f78c6742f74a39a81174f4fc8d01e74813ee733..45f62e78628e7801d80608d729b0e700ffca3960 100644 --- a/tools/ipc-linux.h +++ b/tools/ipc-linux.h @@ -139,15 +139,33 @@ cleanup: return ret; } +static int kernel_get_device(struct wgdevice **device, const char *iface); + static int kernel_set_device(struct wgdevice *dev) { int ret = 0; + struct wgdevice *current = NULL; struct wgpeer *peer = NULL; struct wgallowedip *allowedip = NULL; struct nlattr *peers_nest, *peer_nest, *allowedips_nest, *allowedip_nest; struct nlmsghdr *nlh; struct mnlg_socket *nlg; + if (dev->flags & WGDEVICE_HAS_TRANSPORT) { + ret = kernel_get_device(¤t, dev->name); + if (ret < 0) + return ret; + if (!(current->flags & WGDEVICE_HAS_TRANSPORT)) { + free_wgdevice(current); + if (dev->transport == WG_TRANSPORT_TCP) { + errno = EOPNOTSUPP; + return -EOPNOTSUPP; + } + dev->flags &= ~WGDEVICE_HAS_TRANSPORT; + } else + free_wgdevice(current); + } + nlg = mnlg_socket_open(WG_GENL_NAME, WG_GENL_VERSION); if (!nlg) return -errno; @@ -165,6 +183,8 @@ again: mnl_attr_put_u16(nlh, WGDEVICE_A_LISTEN_PORT, dev->listen_port); if (dev->flags & WGDEVICE_HAS_FWMARK) mnl_attr_put_u32(nlh, WGDEVICE_A_FWMARK, dev->fwmark); + if (dev->flags & WGDEVICE_HAS_TRANSPORT) + mnl_attr_put_u8(nlh, WGDEVICE_A_TRANSPORT, dev->transport); if (dev->flags & WGDEVICE_REPLACE_PEERS) flags |= WGDEVICE_F_REPLACE_PEERS; if (flags) @@ -441,6 +461,14 @@ static int parse_device(const struct nlattr *attr, void *data) if (!mnl_attr_validate(attr, MNL_TYPE_U32)) device->fwmark = mnl_attr_get_u32(attr); break; + case WGDEVICE_A_TRANSPORT: + if (!mnl_attr_validate(attr, MNL_TYPE_U8)) { + device->transport = mnl_attr_get_u8(attr); + if (device->transport > WG_TRANSPORT_TCP) + return MNL_CB_ERROR; + device->flags |= WGDEVICE_HAS_TRANSPORT; + } + break; case WGDEVICE_A_PEERS: return mnl_attr_parse_nested(attr, parse_peers, device); } diff --git a/tools/ipc.c b/tools/ipc.c index 945c9363a4afb415516f8b67cd08c12993ef72a7..b1e01177b11db971f9f1c3c6a7df861c4ae06a0c 100644 --- a/tools/ipc.c +++ b/tools/ipc.c @@ -86,13 +86,30 @@ int ipc_get_device(struct wgdevice **dev, const char *iface) #endif } +static int prepare_userspace_transport(struct wgdevice *dev) +{ + if (!(dev->flags & WGDEVICE_HAS_TRANSPORT)) + return 0; + if (dev->transport == WG_TRANSPORT_TCP) { + errno = EOPNOTSUPP; + return -EOPNOTSUPP; + } + dev->flags &= ~WGDEVICE_HAS_TRANSPORT; + return 0; +} + int ipc_set_device(struct wgdevice *dev) { #ifdef IPC_SUPPORTS_KERNEL_INTERFACE - if (userspace_has_wireguard_interface(dev->name)) + if (userspace_has_wireguard_interface(dev->name)) { + if (prepare_userspace_transport(dev) < 0) + return -EOPNOTSUPP; return userspace_set_device(dev); + } return kernel_set_device(dev); #else + if (prepare_userspace_transport(dev) < 0) + return -EOPNOTSUPP; return userspace_set_device(dev); #endif } diff --git a/tools/netlink.h b/tools/netlink.h index ca11bae47b11f0787e25bb815edd592d1c78bafc..7806772b02f0214204e29507c29166c4746cfff5 100644 --- a/tools/netlink.h +++ b/tools/netlink.h @@ -318,6 +318,11 @@ static void mnl_attr_put(struct nlmsghdr *nlh, uint16_t type, size_t len, memset(mnl_attr_get_payload(attr) + len, 0, pad); } +static void mnl_attr_put_u8(struct nlmsghdr *nlh, uint16_t type, uint8_t data) +{ + mnl_attr_put(nlh, type, sizeof(data), &data); +} + static void mnl_attr_put_u16(struct nlmsghdr *nlh, uint16_t type, uint16_t data) { mnl_attr_put(nlh, type, sizeof(uint16_t), &data); diff --git a/tools/set.c b/tools/set.c index 63803d6825168e9d7bc5ca451258880b0e4ef7c2..918d35dbbb92e8c5419f0cf0107d965f030fcdfe 100644 --- a/tools/set.c +++ b/tools/set.c @@ -18,7 +18,7 @@ int set_main(int argc, const char *argv[]) int ret = 1; if (argc < 3) { - fprintf(stderr, "Usage: %s %s [listen-port ] [fwmark ] [private-key ] [peer [remove] [preshared-key ] [endpoint :] [persistent-keepalive ] [allowed-ips [+|-]/[,[+|-]/]...] ]...\n", PROG_NAME, argv[0]); + fprintf(stderr, "Usage: %s %s [listen-port ] [fwmark ] [private-key ] [transport ] [peer [remove] [preshared-key ] [endpoint :] [persistent-keepalive ] [allowed-ips [+|-]/[,[+|-]/]...] ]...\n", PROG_NAME, argv[0]); return 1; } diff --git a/tools/show.c b/tools/show.c index 0f6eab891cf96e08cf35e1ea851c156e68f888d7..bb84fa1589b5dec3a08fc144850fb104ba9f73fb 100644 --- a/tools/show.c +++ b/tools/show.c @@ -126,6 +126,15 @@ static char *endpoint(const struct sockaddr *addr) return buf; } +static const char *transport(uint8_t value) +{ + if (value == WG_TRANSPORT_TCP) + return "tcp"; + if (value == WG_TRANSPORT_UDP) + return "udp"; + return "unknown"; +} + static size_t pretty_time(char *buf, const size_t len, unsigned long long left) { size_t offset = 0; @@ -202,7 +211,7 @@ static char *bytes(uint64_t b) static const char *COMMAND_NAME; static void show_usage(void) { - fprintf(stderr, "Usage: %s %s { | all | interfaces } [public-key | private-key | listen-port | fwmark | peers | preshared-keys | endpoints | allowed-ips | latest-handshakes | transfer | persistent-keepalive | dump]\n", PROG_NAME, COMMAND_NAME); + fprintf(stderr, "Usage: %s %s { | all | interfaces } [public-key | private-key | listen-port | fwmark | transport | peers | preshared-keys | endpoints | allowed-ips | latest-handshakes | transfer | persistent-keepalive | dump]\n", PROG_NAME, COMMAND_NAME); } static void pretty_print(struct wgdevice *device) @@ -220,6 +229,8 @@ static void pretty_print(struct wgdevice *device) terminal_printf(" " TERMINAL_BOLD "listening port" TERMINAL_RESET ": %u\n", device->listen_port); if (device->fwmark) terminal_printf(" " TERMINAL_BOLD "fwmark" TERMINAL_RESET ": 0x%x\n", device->fwmark); + if (device->transport == WG_TRANSPORT_TCP) + terminal_printf(" " TERMINAL_BOLD "transport" TERMINAL_RESET ": tcp\n"); if (device->first_peer) { sort_peers(device); terminal_printf("\n"); @@ -311,6 +322,10 @@ static bool ugly_print(struct wgdevice *device, const char *param, bool with_int printf("0x%x\n", device->fwmark); else printf("off\n"); + } else if (!strcmp(param, "transport")) { + if (with_interface) + printf("%s\t", device->name); + printf("%s\n", transport(device->transport)); } else if (!strcmp(param, "endpoints")) { for_each_wgpeer(device, peer) { if (with_interface) diff --git a/tools/showconf.c b/tools/showconf.c index be02d7ad306ff6c50dabb09d93ea5431a22fdcae..634f16ef0659361f7300c12a383bdd2c81d9c725 100644 --- a/tools/showconf.c +++ b/tools/showconf.c @@ -46,6 +46,8 @@ int showconf_main(int argc, const char *argv[]) key_to_base64(base64, device->private_key); printf("PrivateKey = %s\n", base64); } + if (device->flags & WGDEVICE_HAS_TRANSPORT) + printf("Transport = %s\n", device->transport == WG_TRANSPORT_TCP ? "tcp" : "udp"); printf("\n"); for_each_wgpeer(device, peer) { key_to_base64(base64, peer->public_key); diff --git a/tools/uapi/linux/linux/wireguard.h b/tools/uapi/linux/linux/wireguard.h index c285d66a4bd81903659998c01a3d7fcbc788079b..b01cb746cef0a0b91b80bc38179e9e7f30e0b1fe 100644 --- a/tools/uapi/linux/linux/wireguard.h +++ b/tools/uapi/linux/linux/wireguard.h @@ -29,6 +29,7 @@ * WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN * WGDEVICE_A_LISTEN_PORT: NLA_U16 * WGDEVICE_A_FWMARK: NLA_U32 + * WGDEVICE_A_TRANSPORT: NLA_U8, WG_TRANSPORT_UDP or WG_TRANSPORT_TCP * WGDEVICE_A_PEERS: NLA_NESTED * 0: NLA_NESTED * WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN @@ -83,6 +84,8 @@ * WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly * WGDEVICE_A_FWMARK: NLA_U32, 0 to disable + * WGDEVICE_A_TRANSPORT: NLA_U8, WG_TRANSPORT_UDP or WG_TRANSPORT_TCP; + * omission preserves the current transport * WGDEVICE_A_PEERS: NLA_NESTED * 0: NLA_NESTED * WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN @@ -140,6 +143,9 @@ #define WG_KEY_LEN 32 +#define WG_TRANSPORT_UDP 0 +#define WG_TRANSPORT_TCP 1 + enum wg_cmd { WG_CMD_GET_DEVICE, WG_CMD_SET_DEVICE, @@ -161,6 +167,7 @@ enum wgdevice_attribute { WGDEVICE_A_LISTEN_PORT, WGDEVICE_A_FWMARK, WGDEVICE_A_PEERS, + WGDEVICE_A_TRANSPORT, __WGDEVICE_A_LAST }; #define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1)