dst.c 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * net/core/dst.c Protocol independent destination cache.
  4. *
  5. * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  6. *
  7. */
  8. #include <linux/bitops.h>
  9. #include <linux/errno.h>
  10. #include <linux/init.h>
  11. #include <linux/kernel.h>
  12. #include <linux/workqueue.h>
  13. #include <linux/mm.h>
  14. #include <linux/module.h>
  15. #include <linux/slab.h>
  16. #include <linux/netdevice.h>
  17. #include <linux/skbuff.h>
  18. #include <linux/string.h>
  19. #include <linux/types.h>
  20. #include <net/net_namespace.h>
  21. #include <linux/sched.h>
  22. #include <linux/prefetch.h>
  23. #include <net/lwtunnel.h>
  24. #include <net/xfrm.h>
  25. #include <net/dst.h>
  26. #include <net/dst_metadata.h>
  27. int dst_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb)
  28. {
  29. kfree_skb(skb);
  30. return 0;
  31. }
  32. EXPORT_SYMBOL(dst_discard_out);
  33. const struct dst_metrics dst_default_metrics = {
  34. /* This initializer is needed to force linker to place this variable
  35. * into const section. Otherwise it might end into bss section.
  36. * We really want to avoid false sharing on this variable, and catch
  37. * any writes on it.
  38. */
  39. .refcnt = REFCOUNT_INIT(1),
  40. };
  41. EXPORT_SYMBOL(dst_default_metrics);
  42. void dst_init(struct dst_entry *dst, struct dst_ops *ops,
  43. struct net_device *dev, int initial_ref, int initial_obsolete,
  44. unsigned short flags)
  45. {
  46. dst->dev = dev;
  47. if (dev)
  48. dev_hold(dev);
  49. dst->ops = ops;
  50. dst_init_metrics(dst, dst_default_metrics.metrics, true);
  51. dst->expires = 0UL;
  52. #ifdef CONFIG_XFRM
  53. dst->xfrm = NULL;
  54. #endif
  55. dst->input = dst_discard;
  56. dst->output = dst_discard_out;
  57. dst->error = 0;
  58. dst->obsolete = initial_obsolete;
  59. dst->header_len = 0;
  60. dst->trailer_len = 0;
  61. #ifdef CONFIG_IP_ROUTE_CLASSID
  62. dst->tclassid = 0;
  63. #endif
  64. dst->lwtstate = NULL;
  65. atomic_set(&dst->__refcnt, initial_ref);
  66. dst->__use = 0;
  67. dst->lastuse = jiffies;
  68. dst->flags = flags;
  69. if (!(flags & DST_NOCOUNT))
  70. dst_entries_add(ops, 1);
  71. }
  72. EXPORT_SYMBOL(dst_init);
  73. void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
  74. int initial_ref, int initial_obsolete, unsigned short flags)
  75. {
  76. struct dst_entry *dst;
  77. if (ops->gc &&
  78. !(flags & DST_NOCOUNT) &&
  79. dst_entries_get_fast(ops) > ops->gc_thresh) {
  80. if (ops->gc(ops)) {
  81. pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
  82. return NULL;
  83. }
  84. }
  85. dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
  86. if (!dst)
  87. return NULL;
  88. dst_init(dst, ops, dev, initial_ref, initial_obsolete, flags);
  89. return dst;
  90. }
  91. EXPORT_SYMBOL(dst_alloc);
  92. struct dst_entry *dst_destroy(struct dst_entry * dst)
  93. {
  94. struct dst_entry *child = NULL;
  95. smp_rmb();
  96. #ifdef CONFIG_XFRM
  97. if (dst->xfrm) {
  98. struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
  99. child = xdst->child;
  100. }
  101. #endif
  102. if (!(dst->flags & DST_NOCOUNT))
  103. dst_entries_add(dst->ops, -1);
  104. if (dst->ops->destroy)
  105. dst->ops->destroy(dst);
  106. if (dst->dev)
  107. dev_put(dst->dev);
  108. lwtstate_put(dst->lwtstate);
  109. if (dst->flags & DST_METADATA)
  110. metadata_dst_free((struct metadata_dst *)dst);
  111. else
  112. kmem_cache_free(dst->ops->kmem_cachep, dst);
  113. dst = child;
  114. if (dst)
  115. dst_release_immediate(dst);
  116. return NULL;
  117. }
  118. EXPORT_SYMBOL(dst_destroy);
  119. static void dst_destroy_rcu(struct rcu_head *head)
  120. {
  121. struct dst_entry *dst = container_of(head, struct dst_entry, rcu_head);
  122. dst = dst_destroy(dst);
  123. }
  124. /* Operations to mark dst as DEAD and clean up the net device referenced
  125. * by dst:
  126. * 1. put the dst under blackhole interface and discard all tx/rx packets
  127. * on this route.
  128. * 2. release the net_device
  129. * This function should be called when removing routes from the fib tree
  130. * in preparation for a NETDEV_DOWN/NETDEV_UNREGISTER event and also to
  131. * make the next dst_ops->check() fail.
  132. */
  133. void dst_dev_put(struct dst_entry *dst)
  134. {
  135. struct net_device *dev = dst->dev;
  136. dst->obsolete = DST_OBSOLETE_DEAD;
  137. if (dst->ops->ifdown)
  138. dst->ops->ifdown(dst, dev, true);
  139. dst->input = dst_discard;
  140. dst->output = dst_discard_out;
  141. dst->dev = blackhole_netdev;
  142. dev_hold(dst->dev);
  143. dev_put(dev);
  144. }
  145. EXPORT_SYMBOL(dst_dev_put);
  146. void dst_release(struct dst_entry *dst)
  147. {
  148. if (dst) {
  149. int newrefcnt;
  150. newrefcnt = atomic_dec_return(&dst->__refcnt);
  151. if (WARN_ONCE(newrefcnt < 0, "dst_release underflow"))
  152. net_warn_ratelimited("%s: dst:%p refcnt:%d\n",
  153. __func__, dst, newrefcnt);
  154. if (!newrefcnt)
  155. call_rcu(&dst->rcu_head, dst_destroy_rcu);
  156. }
  157. }
  158. EXPORT_SYMBOL(dst_release);
  159. void dst_release_immediate(struct dst_entry *dst)
  160. {
  161. if (dst) {
  162. int newrefcnt;
  163. newrefcnt = atomic_dec_return(&dst->__refcnt);
  164. if (WARN_ONCE(newrefcnt < 0, "dst_release_immediate underflow"))
  165. net_warn_ratelimited("%s: dst:%p refcnt:%d\n",
  166. __func__, dst, newrefcnt);
  167. if (!newrefcnt)
  168. dst_destroy(dst);
  169. }
  170. }
  171. EXPORT_SYMBOL(dst_release_immediate);
  172. u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old)
  173. {
  174. struct dst_metrics *p = kmalloc(sizeof(*p), GFP_ATOMIC);
  175. if (p) {
  176. struct dst_metrics *old_p = (struct dst_metrics *)__DST_METRICS_PTR(old);
  177. unsigned long prev, new;
  178. refcount_set(&p->refcnt, 1);
  179. memcpy(p->metrics, old_p->metrics, sizeof(p->metrics));
  180. new = (unsigned long) p;
  181. prev = cmpxchg(&dst->_metrics, old, new);
  182. if (prev != old) {
  183. kfree(p);
  184. p = (struct dst_metrics *)__DST_METRICS_PTR(prev);
  185. if (prev & DST_METRICS_READ_ONLY)
  186. p = NULL;
  187. } else if (prev & DST_METRICS_REFCOUNTED) {
  188. if (refcount_dec_and_test(&old_p->refcnt))
  189. kfree(old_p);
  190. }
  191. }
  192. BUILD_BUG_ON(offsetof(struct dst_metrics, metrics) != 0);
  193. return (u32 *)p;
  194. }
  195. EXPORT_SYMBOL(dst_cow_metrics_generic);
  196. /* Caller asserts that dst_metrics_read_only(dst) is false. */
  197. void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old)
  198. {
  199. unsigned long prev, new;
  200. new = ((unsigned long) &dst_default_metrics) | DST_METRICS_READ_ONLY;
  201. prev = cmpxchg(&dst->_metrics, old, new);
  202. if (prev == old)
  203. kfree(__DST_METRICS_PTR(old));
  204. }
  205. EXPORT_SYMBOL(__dst_destroy_metrics_generic);
  206. struct dst_entry *dst_blackhole_check(struct dst_entry *dst, u32 cookie)
  207. {
  208. return NULL;
  209. }
  210. u32 *dst_blackhole_cow_metrics(struct dst_entry *dst, unsigned long old)
  211. {
  212. return NULL;
  213. }
  214. struct neighbour *dst_blackhole_neigh_lookup(const struct dst_entry *dst,
  215. struct sk_buff *skb,
  216. const void *daddr)
  217. {
  218. return NULL;
  219. }
  220. void dst_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
  221. struct sk_buff *skb, u32 mtu,
  222. bool confirm_neigh)
  223. {
  224. }
  225. EXPORT_SYMBOL_GPL(dst_blackhole_update_pmtu);
  226. void dst_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
  227. struct sk_buff *skb)
  228. {
  229. }
  230. EXPORT_SYMBOL_GPL(dst_blackhole_redirect);
  231. unsigned int dst_blackhole_mtu(const struct dst_entry *dst)
  232. {
  233. unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
  234. return mtu ? : dst->dev->mtu;
  235. }
  236. EXPORT_SYMBOL_GPL(dst_blackhole_mtu);
  237. static struct dst_ops dst_blackhole_ops = {
  238. .family = AF_UNSPEC,
  239. .neigh_lookup = dst_blackhole_neigh_lookup,
  240. .check = dst_blackhole_check,
  241. .cow_metrics = dst_blackhole_cow_metrics,
  242. .update_pmtu = dst_blackhole_update_pmtu,
  243. .redirect = dst_blackhole_redirect,
  244. .mtu = dst_blackhole_mtu,
  245. };
  246. static void __metadata_dst_init(struct metadata_dst *md_dst,
  247. enum metadata_type type, u8 optslen)
  248. {
  249. struct dst_entry *dst;
  250. dst = &md_dst->dst;
  251. dst_init(dst, &dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE,
  252. DST_METADATA | DST_NOCOUNT);
  253. memset(dst + 1, 0, sizeof(*md_dst) + optslen - sizeof(*dst));
  254. md_dst->type = type;
  255. }
  256. struct metadata_dst *metadata_dst_alloc(u8 optslen, enum metadata_type type,
  257. gfp_t flags)
  258. {
  259. struct metadata_dst *md_dst;
  260. md_dst = kmalloc(sizeof(*md_dst) + optslen, flags);
  261. if (!md_dst)
  262. return NULL;
  263. __metadata_dst_init(md_dst, type, optslen);
  264. return md_dst;
  265. }
  266. EXPORT_SYMBOL_GPL(metadata_dst_alloc);
  267. void metadata_dst_free(struct metadata_dst *md_dst)
  268. {
  269. #ifdef CONFIG_DST_CACHE
  270. if (md_dst->type == METADATA_IP_TUNNEL)
  271. dst_cache_destroy(&md_dst->u.tun_info.dst_cache);
  272. #endif
  273. kfree(md_dst);
  274. }
  275. EXPORT_SYMBOL_GPL(metadata_dst_free);
  276. struct metadata_dst __percpu *
  277. metadata_dst_alloc_percpu(u8 optslen, enum metadata_type type, gfp_t flags)
  278. {
  279. int cpu;
  280. struct metadata_dst __percpu *md_dst;
  281. md_dst = __alloc_percpu_gfp(sizeof(struct metadata_dst) + optslen,
  282. __alignof__(struct metadata_dst), flags);
  283. if (!md_dst)
  284. return NULL;
  285. for_each_possible_cpu(cpu)
  286. __metadata_dst_init(per_cpu_ptr(md_dst, cpu), type, optslen);
  287. return md_dst;
  288. }
  289. EXPORT_SYMBOL_GPL(metadata_dst_alloc_percpu);
  290. void metadata_dst_free_percpu(struct metadata_dst __percpu *md_dst)
  291. {
  292. #ifdef CONFIG_DST_CACHE
  293. int cpu;
  294. for_each_possible_cpu(cpu) {
  295. struct metadata_dst *one_md_dst = per_cpu_ptr(md_dst, cpu);
  296. if (one_md_dst->type == METADATA_IP_TUNNEL)
  297. dst_cache_destroy(&one_md_dst->u.tun_info.dst_cache);
  298. }
  299. #endif
  300. free_percpu(md_dst);
  301. }
  302. EXPORT_SYMBOL_GPL(metadata_dst_free_percpu);