device.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  4. */
  5. #include "queueing.h"
  6. #include "socket.h"
  7. #include "timers.h"
  8. #include "device.h"
  9. #include "ratelimiter.h"
  10. #include "peer.h"
  11. #include "messages.h"
  12. #include <linux/module.h>
  13. #include <linux/rtnetlink.h>
  14. #include <linux/inet.h>
  15. #include <linux/netdevice.h>
  16. #include <linux/inetdevice.h>
  17. #include <linux/if_arp.h>
  18. #include <linux/icmp.h>
  19. #include <linux/suspend.h>
  20. #include <net/icmp.h>
  21. #include <net/rtnetlink.h>
  22. #include <net/ip_tunnels.h>
  23. #include <net/addrconf.h>
  24. static LIST_HEAD(device_list);
  25. static int wg_open(struct net_device *dev)
  26. {
  27. struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
  28. struct inet6_dev *dev_v6 = __in6_dev_get(dev);
  29. struct wg_device *wg = netdev_priv(dev);
  30. struct wg_peer *peer;
  31. int ret;
  32. if (dev_v4) {
  33. /* At some point we might put this check near the ip_rt_send_
  34. * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
  35. * to the current secpath check.
  36. */
  37. IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
  38. IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
  39. }
  40. if (dev_v6)
  41. dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
  42. mutex_lock(&wg->device_update_lock);
  43. ret = wg_socket_init(wg, wg->incoming_port);
  44. if (ret < 0)
  45. goto out;
  46. list_for_each_entry(peer, &wg->peer_list, peer_list) {
  47. wg_packet_send_staged_packets(peer);
  48. if (peer->persistent_keepalive_interval)
  49. wg_packet_send_keepalive(peer);
  50. }
  51. out:
  52. mutex_unlock(&wg->device_update_lock);
  53. return ret;
  54. }
  55. #ifdef CONFIG_PM_SLEEP
  56. static int wg_pm_notification(struct notifier_block *nb, unsigned long action,
  57. void *data)
  58. {
  59. struct wg_device *wg;
  60. struct wg_peer *peer;
  61. /* If the machine is constantly suspending and resuming, as part of
  62. * its normal operation rather than as a somewhat rare event, then we
  63. * don't actually want to clear keys.
  64. */
  65. if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID))
  66. return 0;
  67. if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
  68. return 0;
  69. rtnl_lock();
  70. list_for_each_entry(wg, &device_list, device_list) {
  71. mutex_lock(&wg->device_update_lock);
  72. list_for_each_entry(peer, &wg->peer_list, peer_list) {
  73. del_timer(&peer->timer_zero_key_material);
  74. wg_noise_handshake_clear(&peer->handshake);
  75. wg_noise_keypairs_clear(&peer->keypairs);
  76. }
  77. mutex_unlock(&wg->device_update_lock);
  78. }
  79. rtnl_unlock();
  80. rcu_barrier();
  81. return 0;
  82. }
  83. static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
  84. #endif
  85. static int wg_stop(struct net_device *dev)
  86. {
  87. struct wg_device *wg = netdev_priv(dev);
  88. struct wg_peer *peer;
  89. struct sk_buff *skb;
  90. mutex_lock(&wg->device_update_lock);
  91. list_for_each_entry(peer, &wg->peer_list, peer_list) {
  92. wg_packet_purge_staged_packets(peer);
  93. wg_timers_stop(peer);
  94. wg_noise_handshake_clear(&peer->handshake);
  95. wg_noise_keypairs_clear(&peer->keypairs);
  96. wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
  97. }
  98. mutex_unlock(&wg->device_update_lock);
  99. while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL)
  100. kfree_skb(skb);
  101. atomic_set(&wg->handshake_queue_len, 0);
  102. wg_socket_reinit(wg, NULL, NULL);
  103. return 0;
  104. }
  105. static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
  106. {
  107. struct wg_device *wg = netdev_priv(dev);
  108. struct sk_buff_head packets;
  109. struct wg_peer *peer;
  110. struct sk_buff *next;
  111. sa_family_t family;
  112. u32 mtu;
  113. int ret;
  114. if (unlikely(!wg_check_packet_protocol(skb))) {
  115. ret = -EPROTONOSUPPORT;
  116. net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
  117. goto err;
  118. }
  119. peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
  120. if (unlikely(!peer)) {
  121. ret = -ENOKEY;
  122. if (skb->protocol == htons(ETH_P_IP))
  123. net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
  124. dev->name, &ip_hdr(skb)->daddr);
  125. else if (skb->protocol == htons(ETH_P_IPV6))
  126. net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
  127. dev->name, &ipv6_hdr(skb)->daddr);
  128. goto err_icmp;
  129. }
  130. family = READ_ONCE(peer->endpoint.addr.sa_family);
  131. if (unlikely(family != AF_INET && family != AF_INET6)) {
  132. ret = -EDESTADDRREQ;
  133. net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
  134. dev->name, peer->internal_id);
  135. goto err_peer;
  136. }
  137. mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
  138. __skb_queue_head_init(&packets);
  139. if (!skb_is_gso(skb)) {
  140. skb_mark_not_on_list(skb);
  141. } else {
  142. struct sk_buff *segs = skb_gso_segment(skb, 0);
  143. if (unlikely(IS_ERR(segs))) {
  144. ret = PTR_ERR(segs);
  145. goto err_peer;
  146. }
  147. dev_kfree_skb(skb);
  148. skb = segs;
  149. }
  150. skb_list_walk_safe(skb, skb, next) {
  151. skb_mark_not_on_list(skb);
  152. skb = skb_share_check(skb, GFP_ATOMIC);
  153. if (unlikely(!skb))
  154. continue;
  155. /* We only need to keep the original dst around for icmp,
  156. * so at this point we're in a position to drop it.
  157. */
  158. skb_dst_drop(skb);
  159. PACKET_CB(skb)->mtu = mtu;
  160. __skb_queue_tail(&packets, skb);
  161. }
  162. spin_lock_bh(&peer->staged_packet_queue.lock);
  163. /* If the queue is getting too big, we start removing the oldest packets
  164. * until it's small again. We do this before adding the new packet, so
  165. * we don't remove GSO segments that are in excess.
  166. */
  167. while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
  168. dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
  169. ++dev->stats.tx_dropped;
  170. }
  171. skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
  172. spin_unlock_bh(&peer->staged_packet_queue.lock);
  173. wg_packet_send_staged_packets(peer);
  174. wg_peer_put(peer);
  175. return NETDEV_TX_OK;
  176. err_peer:
  177. wg_peer_put(peer);
  178. err_icmp:
  179. if (skb->protocol == htons(ETH_P_IP))
  180. icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
  181. else if (skb->protocol == htons(ETH_P_IPV6))
  182. icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
  183. err:
  184. ++dev->stats.tx_errors;
  185. kfree_skb(skb);
  186. return ret;
  187. }
  188. static const struct net_device_ops netdev_ops = {
  189. .ndo_open = wg_open,
  190. .ndo_stop = wg_stop,
  191. .ndo_start_xmit = wg_xmit,
  192. .ndo_get_stats64 = ip_tunnel_get_stats64
  193. };
  194. static void wg_destruct(struct net_device *dev)
  195. {
  196. struct wg_device *wg = netdev_priv(dev);
  197. rtnl_lock();
  198. list_del(&wg->device_list);
  199. rtnl_unlock();
  200. mutex_lock(&wg->device_update_lock);
  201. rcu_assign_pointer(wg->creating_net, NULL);
  202. wg->incoming_port = 0;
  203. wg_socket_reinit(wg, NULL, NULL);
  204. /* The final references are cleared in the below calls to destroy_workqueue. */
  205. wg_peer_remove_all(wg);
  206. destroy_workqueue(wg->handshake_receive_wq);
  207. destroy_workqueue(wg->handshake_send_wq);
  208. destroy_workqueue(wg->packet_crypt_wq);
  209. wg_packet_queue_free(&wg->handshake_queue, true);
  210. wg_packet_queue_free(&wg->decrypt_queue, false);
  211. wg_packet_queue_free(&wg->encrypt_queue, false);
  212. rcu_barrier(); /* Wait for all the peers to be actually freed. */
  213. wg_ratelimiter_uninit();
  214. memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
  215. free_percpu(dev->tstats);
  216. kvfree(wg->index_hashtable);
  217. kvfree(wg->peer_hashtable);
  218. mutex_unlock(&wg->device_update_lock);
  219. pr_debug("%s: Interface destroyed\n", dev->name);
  220. free_netdev(dev);
  221. }
  222. static const struct device_type device_type = { .name = KBUILD_MODNAME };
  223. static void wg_setup(struct net_device *dev)
  224. {
  225. struct wg_device *wg = netdev_priv(dev);
  226. enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
  227. NETIF_F_SG | NETIF_F_GSO |
  228. NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
  229. const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
  230. max(sizeof(struct ipv6hdr), sizeof(struct iphdr));
  231. dev->netdev_ops = &netdev_ops;
  232. dev->header_ops = &ip_tunnel_header_ops;
  233. dev->hard_header_len = 0;
  234. dev->addr_len = 0;
  235. dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
  236. dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
  237. dev->type = ARPHRD_NONE;
  238. dev->flags = IFF_POINTOPOINT | IFF_NOARP;
  239. dev->priv_flags |= IFF_NO_QUEUE;
  240. dev->features |= NETIF_F_LLTX;
  241. dev->features |= WG_NETDEV_FEATURES;
  242. dev->hw_features |= WG_NETDEV_FEATURES;
  243. dev->hw_enc_features |= WG_NETDEV_FEATURES;
  244. dev->mtu = ETH_DATA_LEN - overhead;
  245. dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
  246. SET_NETDEV_DEVTYPE(dev, &device_type);
  247. /* We need to keep the dst around in case of icmp replies. */
  248. netif_keep_dst(dev);
  249. memset(wg, 0, sizeof(*wg));
  250. wg->dev = dev;
  251. }
  252. static int wg_newlink(struct net *src_net, struct net_device *dev,
  253. struct nlattr *tb[], struct nlattr *data[],
  254. struct netlink_ext_ack *extack)
  255. {
  256. struct wg_device *wg = netdev_priv(dev);
  257. int ret = -ENOMEM;
  258. rcu_assign_pointer(wg->creating_net, src_net);
  259. init_rwsem(&wg->static_identity.lock);
  260. mutex_init(&wg->socket_update_lock);
  261. mutex_init(&wg->device_update_lock);
  262. wg_allowedips_init(&wg->peer_allowedips);
  263. wg_cookie_checker_init(&wg->cookie_checker, wg);
  264. INIT_LIST_HEAD(&wg->peer_list);
  265. wg->device_update_gen = 1;
  266. wg->peer_hashtable = wg_pubkey_hashtable_alloc();
  267. if (!wg->peer_hashtable)
  268. return ret;
  269. wg->index_hashtable = wg_index_hashtable_alloc();
  270. if (!wg->index_hashtable)
  271. goto err_free_peer_hashtable;
  272. dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
  273. if (!dev->tstats)
  274. goto err_free_index_hashtable;
  275. wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
  276. WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
  277. if (!wg->handshake_receive_wq)
  278. goto err_free_tstats;
  279. wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
  280. WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
  281. if (!wg->handshake_send_wq)
  282. goto err_destroy_handshake_receive;
  283. wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
  284. WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
  285. if (!wg->packet_crypt_wq)
  286. goto err_destroy_handshake_send;
  287. ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
  288. MAX_QUEUED_PACKETS);
  289. if (ret < 0)
  290. goto err_destroy_packet_crypt;
  291. ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
  292. MAX_QUEUED_PACKETS);
  293. if (ret < 0)
  294. goto err_free_encrypt_queue;
  295. ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker,
  296. MAX_QUEUED_INCOMING_HANDSHAKES);
  297. if (ret < 0)
  298. goto err_free_decrypt_queue;
  299. ret = wg_ratelimiter_init();
  300. if (ret < 0)
  301. goto err_free_handshake_queue;
  302. ret = register_netdevice(dev);
  303. if (ret < 0)
  304. goto err_uninit_ratelimiter;
  305. list_add(&wg->device_list, &device_list);
  306. /* We wait until the end to assign priv_destructor, so that
  307. * register_netdevice doesn't call it for us if it fails.
  308. */
  309. dev->priv_destructor = wg_destruct;
  310. pr_debug("%s: Interface created\n", dev->name);
  311. return ret;
  312. err_uninit_ratelimiter:
  313. wg_ratelimiter_uninit();
  314. err_free_handshake_queue:
  315. wg_packet_queue_free(&wg->handshake_queue, false);
  316. err_free_decrypt_queue:
  317. wg_packet_queue_free(&wg->decrypt_queue, false);
  318. err_free_encrypt_queue:
  319. wg_packet_queue_free(&wg->encrypt_queue, false);
  320. err_destroy_packet_crypt:
  321. destroy_workqueue(wg->packet_crypt_wq);
  322. err_destroy_handshake_send:
  323. destroy_workqueue(wg->handshake_send_wq);
  324. err_destroy_handshake_receive:
  325. destroy_workqueue(wg->handshake_receive_wq);
  326. err_free_tstats:
  327. free_percpu(dev->tstats);
  328. err_free_index_hashtable:
  329. kvfree(wg->index_hashtable);
  330. err_free_peer_hashtable:
  331. kvfree(wg->peer_hashtable);
  332. return ret;
  333. }
  334. static struct rtnl_link_ops link_ops __read_mostly = {
  335. .kind = KBUILD_MODNAME,
  336. .priv_size = sizeof(struct wg_device),
  337. .setup = wg_setup,
  338. .newlink = wg_newlink,
  339. };
  340. static void wg_netns_pre_exit(struct net *net)
  341. {
  342. struct wg_device *wg;
  343. struct wg_peer *peer;
  344. rtnl_lock();
  345. list_for_each_entry(wg, &device_list, device_list) {
  346. if (rcu_access_pointer(wg->creating_net) == net) {
  347. pr_debug("%s: Creating namespace exiting\n", wg->dev->name);
  348. netif_carrier_off(wg->dev);
  349. mutex_lock(&wg->device_update_lock);
  350. rcu_assign_pointer(wg->creating_net, NULL);
  351. wg_socket_reinit(wg, NULL, NULL);
  352. list_for_each_entry(peer, &wg->peer_list, peer_list)
  353. wg_socket_clear_peer_endpoint_src(peer);
  354. mutex_unlock(&wg->device_update_lock);
  355. }
  356. }
  357. rtnl_unlock();
  358. }
  359. static struct pernet_operations pernet_ops = {
  360. .pre_exit = wg_netns_pre_exit
  361. };
  362. int __init wg_device_init(void)
  363. {
  364. int ret;
  365. #ifdef CONFIG_PM_SLEEP
  366. ret = register_pm_notifier(&pm_notifier);
  367. if (ret)
  368. return ret;
  369. #endif
  370. ret = register_pernet_device(&pernet_ops);
  371. if (ret)
  372. goto error_pm;
  373. ret = rtnl_link_register(&link_ops);
  374. if (ret)
  375. goto error_pernet;
  376. return 0;
  377. error_pernet:
  378. unregister_pernet_device(&pernet_ops);
  379. error_pm:
  380. #ifdef CONFIG_PM_SLEEP
  381. unregister_pm_notifier(&pm_notifier);
  382. #endif
  383. return ret;
  384. }
  385. void wg_device_uninit(void)
  386. {
  387. rtnl_link_unregister(&link_ops);
  388. unregister_pernet_device(&pernet_ops);
  389. #ifdef CONFIG_PM_SLEEP
  390. unregister_pm_notifier(&pm_notifier);
  391. #endif
  392. rcu_barrier();
  393. }