skbuff.c 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041
  1. /*
  2. * Routines having to do with the 'struct sk_buff' memory handlers.
  3. *
  4. * Authors: Alan Cox <iiitac@pyr.swan.ac.uk>
  5. * Florian La Roche <rzsfl@rz.uni-sb.de>
  6. *
  7. * Version: $Id: skbuff.c,v 1.1.1.1 2007/06/12 07:27:14 eyryu Exp $
  8. *
  9. * Fixes:
  10. * Alan Cox : Fixed the worst of the load
  11. * balancer bugs.
  12. * Dave Platt : Interrupt stacking fix.
  13. * Richard Kooijman : Timestamp fixes.
  14. * Alan Cox : Changed buffer format.
  15. * Alan Cox : destructor hook for AF_UNIX etc.
  16. * Linus Torvalds : Better skb_clone.
  17. * Alan Cox : Added skb_copy.
  18. * Alan Cox : Added all the changed routines Linus
  19. * only put in the headers
  20. * Ray VanTassle : Fixed --skb->lock in free
  21. * Alan Cox : skb_copy copy arp field
  22. * Andi Kleen : slabified it.
  23. * Robert Olsson : Removed skb_head_pool
  24. *
  25. * NOTE:
  26. * The __skb_ routines should be called with interrupts
  27. * disabled, or you better be *real* sure that the operation is atomic
  28. * with respect to whatever list is being frobbed (e.g. via lock_sock()
  29. * or via disabling bottom half handlers, etc).
  30. *
  31. * This program is free software; you can redistribute it and/or
  32. * modify it under the terms of the GNU General Public License
  33. * as published by the Free Software Foundation; either version
  34. * 2 of the License, or (at your option) any later version.
  35. */
  36. /*
  37. * The functions in this file will not compile correctly with gcc 2.4.x
  38. */
  39. #include <linux/module.h>
  40. #include <linux/types.h>
  41. #include <linux/kernel.h>
  42. #include <linux/mm.h>
  43. #include <linux/interrupt.h>
  44. #include <linux/in.h>
  45. #include <linux/inet.h>
  46. #include <linux/slab.h>
  47. #include <linux/netdevice.h>
  48. #ifdef CONFIG_NET_CLS_ACT
  49. #include <net/pkt_sched.h>
  50. #endif
  51. #include <linux/string.h>
  52. #include <linux/skbuff.h>
  53. #include <linux/cache.h>
  54. #include <linux/rtnetlink.h>
  55. #include <linux/init.h>
  56. #include <net/protocol.h>
  57. #include <net/dst.h>
  58. #include <net/sock.h>
  59. #include <net/checksum.h>
  60. #include <net/xfrm.h>
  61. #include <asm/uaccess.h>
  62. #include <asm/system.h>
  63. #include "kmap_skb.h"
  64. static struct kmem_cache *skbuff_head_cache __read_mostly;
  65. static struct kmem_cache *skbuff_fclone_cache __read_mostly;
  66. /*
  67. * Keep out-of-line to prevent kernel bloat.
  68. * __builtin_return_address is not used because it is not always
  69. * reliable.
  70. */
  71. /**
  72. * skb_over_panic - private function
  73. * @skb: buffer
  74. * @sz: size
  75. * @here: address
  76. *
  77. * Out of line support code for skb_put(). Not user callable.
  78. */
  79. void skb_over_panic(struct sk_buff *skb, int sz, void *here)
  80. {
  81. printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
  82. "data:%p tail:%p end:%p dev:%s\n",
  83. here, skb->len, sz, skb->head, skb->data, skb->tail, skb->end,
  84. skb->dev ? skb->dev->name : "<NULL>");
  85. BUG();
  86. }
  87. /**
  88. * skb_under_panic - private function
  89. * @skb: buffer
  90. * @sz: size
  91. * @here: address
  92. *
  93. * Out of line support code for skb_push(). Not user callable.
  94. */
  95. void skb_under_panic(struct sk_buff *skb, int sz, void *here)
  96. {
  97. printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
  98. "data:%p tail:%p end:%p dev:%s\n",
  99. here, skb->len, sz, skb->head, skb->data, skb->tail, skb->end,
  100. skb->dev ? skb->dev->name : "<NULL>");
  101. BUG();
  102. }
  103. void skb_truesize_bug(struct sk_buff *skb)
  104. {
  105. printk(KERN_ERR "SKB BUG: Invalid truesize (%u) "
  106. "len=%u, sizeof(sk_buff)=%Zd\n",
  107. skb->truesize, skb->len, sizeof(struct sk_buff));
  108. }
  109. EXPORT_SYMBOL(skb_truesize_bug);
  110. /* Allocate a new skbuff. We do this ourselves so we can fill in a few
  111. * 'private' fields and also do memory statistics to find all the
  112. * [BEEP] leaks.
  113. *
  114. */
  115. /**
  116. * __alloc_skb - allocate a network buffer
  117. * @size: size to allocate
  118. * @gfp_mask: allocation mask
  119. * @fclone: allocate from fclone cache instead of head cache
  120. * and allocate a cloned (child) skb
  121. * @node: numa node to allocate memory on
  122. *
  123. * Allocate a new &sk_buff. The returned buffer has no headroom and a
  124. * tail room of size bytes. The object has a reference count of one.
  125. * The return is the buffer. On a failure the return is %NULL.
  126. *
  127. * Buffers may only be allocated from interrupts using a @gfp_mask of
  128. * %GFP_ATOMIC.
  129. */
  130. struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
  131. int fclone, int node)
  132. {
  133. struct kmem_cache *cache;
  134. struct skb_shared_info *shinfo;
  135. struct sk_buff *skb;
  136. u8 *data;
  137. cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
  138. /* Get the HEAD */
  139. skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
  140. if (!skb)
  141. goto out;
  142. /* Get the DATA. Size must match skb_add_mtu(). */
  143. size = SKB_DATA_ALIGN(size);
  144. data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
  145. gfp_mask, node);
  146. if (!data)
  147. goto nodata;
  148. memset(skb, 0, offsetof(struct sk_buff, truesize));
  149. skb->truesize = size + sizeof(struct sk_buff);
  150. atomic_set(&skb->users, 1);
  151. skb->head = data;
  152. skb->data = data;
  153. skb->tail = data;
  154. skb->end = data + size;
  155. /* make sure we initialize shinfo sequentially */
  156. shinfo = skb_shinfo(skb);
  157. atomic_set(&shinfo->dataref, 1);
  158. shinfo->nr_frags = 0;
  159. shinfo->gso_size = 0;
  160. shinfo->gso_segs = 0;
  161. shinfo->gso_type = 0;
  162. shinfo->ip6_frag_id = 0;
  163. shinfo->frag_list = NULL;
  164. if (fclone) {
  165. struct sk_buff *child = skb + 1;
  166. atomic_t *fclone_ref = (atomic_t *) (child + 1);
  167. skb->fclone = SKB_FCLONE_ORIG;
  168. atomic_set(fclone_ref, 1);
  169. child->fclone = SKB_FCLONE_UNAVAILABLE;
  170. }
  171. out:
  172. return skb;
  173. nodata:
  174. kmem_cache_free(cache, skb);
  175. skb = NULL;
  176. goto out;
  177. }
  178. /**
  179. * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
  180. * @dev: network device to receive on
  181. * @length: length to allocate
  182. * @gfp_mask: get_free_pages mask, passed to alloc_skb
  183. *
  184. * Allocate a new &sk_buff and assign it a usage count of one. The
  185. * buffer has unspecified headroom built in. Users should allocate
  186. * the headroom they think they need without accounting for the
  187. * built in space. The built in space is used for optimisations.
  188. *
  189. * %NULL is returned if there is no free memory.
  190. */
  191. struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
  192. unsigned int length, gfp_t gfp_mask)
  193. {
  194. int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
  195. struct sk_buff *skb;
  196. skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);
  197. if (likely(skb)) {
  198. skb_reserve(skb, NET_SKB_PAD);
  199. skb->dev = dev;
  200. }
  201. return skb;
  202. }
  203. static void skb_drop_list(struct sk_buff **listp)
  204. {
  205. struct sk_buff *list = *listp;
  206. *listp = NULL;
  207. do {
  208. struct sk_buff *this = list;
  209. list = list->next;
  210. kfree_skb(this);
  211. } while (list);
  212. }
  213. static inline void skb_drop_fraglist(struct sk_buff *skb)
  214. {
  215. skb_drop_list(&skb_shinfo(skb)->frag_list);
  216. }
  217. static void skb_clone_fraglist(struct sk_buff *skb)
  218. {
  219. struct sk_buff *list;
  220. for (list = skb_shinfo(skb)->frag_list; list; list = list->next)
  221. skb_get(list);
  222. }
  223. static void skb_release_data(struct sk_buff *skb)
  224. {
  225. if (!skb->cloned ||
  226. !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
  227. &skb_shinfo(skb)->dataref)) {
  228. if (skb_shinfo(skb)->nr_frags) {
  229. int i;
  230. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  231. put_page(skb_shinfo(skb)->frags[i].page);
  232. }
  233. if (skb_shinfo(skb)->frag_list)
  234. skb_drop_fraglist(skb);
  235. kfree(skb->head);
  236. }
  237. }
  238. /*
  239. * Free an skbuff by memory without cleaning the state.
  240. */
  241. void kfree_skbmem(struct sk_buff *skb)
  242. {
  243. struct sk_buff *other;
  244. atomic_t *fclone_ref;
  245. skb_release_data(skb);
  246. switch (skb->fclone) {
  247. case SKB_FCLONE_UNAVAILABLE:
  248. kmem_cache_free(skbuff_head_cache, skb);
  249. break;
  250. case SKB_FCLONE_ORIG:
  251. fclone_ref = (atomic_t *) (skb + 2);
  252. if (atomic_dec_and_test(fclone_ref))
  253. kmem_cache_free(skbuff_fclone_cache, skb);
  254. break;
  255. case SKB_FCLONE_CLONE:
  256. fclone_ref = (atomic_t *) (skb + 1);
  257. other = skb - 1;
  258. /* The clone portion is available for
  259. * fast-cloning again.
  260. */
  261. skb->fclone = SKB_FCLONE_UNAVAILABLE;
  262. if (atomic_dec_and_test(fclone_ref))
  263. kmem_cache_free(skbuff_fclone_cache, other);
  264. break;
  265. };
  266. }
  267. /**
  268. * __kfree_skb - private function
  269. * @skb: buffer
  270. *
  271. * Free an sk_buff. Release anything attached to the buffer.
  272. * Clean the state. This is an internal helper function. Users should
  273. * always call kfree_skb
  274. */
  275. void __kfree_skb(struct sk_buff *skb)
  276. {
  277. dst_release(skb->dst);
  278. #ifdef CONFIG_XFRM
  279. secpath_put(skb->sp);
  280. #endif
  281. if (skb->destructor) {
  282. WARN_ON(in_irq());
  283. skb->destructor(skb);
  284. }
  285. #ifdef CONFIG_NETFILTER
  286. nf_conntrack_put(skb->nfct);
  287. #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
  288. nf_conntrack_put_reasm(skb->nfct_reasm);
  289. #endif
  290. #ifdef CONFIG_BRIDGE_NETFILTER
  291. nf_bridge_put(skb->nf_bridge);
  292. #endif
  293. #endif
  294. /* XXX: IS this still necessary? - JHS */
  295. #ifdef CONFIG_NET_SCHED
  296. skb->tc_index = 0;
  297. #ifdef CONFIG_NET_CLS_ACT
  298. skb->tc_verd = 0;
  299. #endif
  300. #endif
  301. kfree_skbmem(skb);
  302. }
  303. /**
  304. * kfree_skb - free an sk_buff
  305. * @skb: buffer to free
  306. *
  307. * Drop a reference to the buffer and free it if the usage count has
  308. * hit zero.
  309. */
  310. void kfree_skb(struct sk_buff *skb)
  311. {
  312. if (unlikely(!skb))
  313. return;
  314. if (likely(atomic_read(&skb->users) == 1))
  315. smp_rmb();
  316. else if (likely(!atomic_dec_and_test(&skb->users)))
  317. return;
  318. __kfree_skb(skb);
  319. }
  320. /**
  321. * skb_clone - duplicate an sk_buff
  322. * @skb: buffer to clone
  323. * @gfp_mask: allocation priority
  324. *
  325. * Duplicate an &sk_buff. The new one is not owned by a socket. Both
  326. * copies share the same packet data but not structure. The new
  327. * buffer has a reference count of 1. If the allocation fails the
  328. * function returns %NULL otherwise the new buffer is returned.
  329. *
  330. * If this function is called from an interrupt gfp_mask() must be
  331. * %GFP_ATOMIC.
  332. */
  333. struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
  334. {
  335. struct sk_buff *n;
  336. n = skb + 1;
  337. if (skb->fclone == SKB_FCLONE_ORIG &&
  338. n->fclone == SKB_FCLONE_UNAVAILABLE) {
  339. atomic_t *fclone_ref = (atomic_t *) (n + 1);
  340. n->fclone = SKB_FCLONE_CLONE;
  341. atomic_inc(fclone_ref);
  342. } else {
  343. n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
  344. if (!n)
  345. return NULL;
  346. n->fclone = SKB_FCLONE_UNAVAILABLE;
  347. }
  348. #define C(x) n->x = skb->x
  349. n->next = n->prev = NULL;
  350. n->sk = NULL;
  351. C(tstamp);
  352. C(dev);
  353. C(h);
  354. C(nh);
  355. C(mac);
  356. C(dst);
  357. dst_clone(skb->dst);
  358. C(sp);
  359. #ifdef CONFIG_INET
  360. secpath_get(skb->sp);
  361. #endif
  362. memcpy(n->cb, skb->cb, sizeof(skb->cb));
  363. C(len);
  364. C(data_len);
  365. C(mac_len);
  366. C(csum);
  367. C(local_df);
  368. n->cloned = 1;
  369. n->nohdr = 0;
  370. C(pkt_type);
  371. C(ip_summed);
  372. C(priority);
  373. #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
  374. C(ipvs_property);
  375. #endif
  376. C(protocol);
  377. n->destructor = NULL;
  378. C(mark);
  379. #ifdef CONFIG_NETFILTER
  380. C(nfct);
  381. nf_conntrack_get(skb->nfct);
  382. C(nfctinfo);
  383. #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
  384. C(nfct_reasm);
  385. nf_conntrack_get_reasm(skb->nfct_reasm);
  386. #endif
  387. #ifdef CONFIG_BRIDGE_NETFILTER
  388. C(nf_bridge);
  389. nf_bridge_get(skb->nf_bridge);
  390. #endif
  391. #endif /*CONFIG_NETFILTER*/
  392. #ifdef CONFIG_NET_SCHED
  393. C(tc_index);
  394. #ifdef CONFIG_NET_CLS_ACT
  395. n->tc_verd = SET_TC_VERD(skb->tc_verd,0);
  396. n->tc_verd = CLR_TC_OK2MUNGE(n->tc_verd);
  397. n->tc_verd = CLR_TC_MUNGED(n->tc_verd);
  398. C(iif);
  399. #endif
  400. skb_copy_secmark(n, skb);
  401. #endif
  402. C(truesize);
  403. atomic_set(&n->users, 1);
  404. C(head);
  405. C(data);
  406. C(tail);
  407. C(end);
  408. atomic_inc(&(skb_shinfo(skb)->dataref));
  409. skb->cloned = 1;
  410. return n;
  411. }
  412. static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  413. {
  414. /*
  415. * Shift between the two data areas in bytes
  416. */
  417. unsigned long offset = new->data - old->data;
  418. new->sk = NULL;
  419. new->dev = old->dev;
  420. new->priority = old->priority;
  421. new->protocol = old->protocol;
  422. new->dst = dst_clone(old->dst);
  423. #ifdef CONFIG_INET
  424. new->sp = secpath_get(old->sp);
  425. #endif
  426. new->h.raw = old->h.raw + offset;
  427. new->nh.raw = old->nh.raw + offset;
  428. new->mac.raw = old->mac.raw + offset;
  429. memcpy(new->cb, old->cb, sizeof(old->cb));
  430. new->local_df = old->local_df;
  431. new->fclone = SKB_FCLONE_UNAVAILABLE;
  432. new->pkt_type = old->pkt_type;
  433. new->tstamp = old->tstamp;
  434. new->destructor = NULL;
  435. new->mark = old->mark;
  436. #ifdef CONFIG_NETFILTER
  437. new->nfct = old->nfct;
  438. nf_conntrack_get(old->nfct);
  439. new->nfctinfo = old->nfctinfo;
  440. #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
  441. new->nfct_reasm = old->nfct_reasm;
  442. nf_conntrack_get_reasm(old->nfct_reasm);
  443. #endif
  444. #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
  445. new->ipvs_property = old->ipvs_property;
  446. #endif
  447. #ifdef CONFIG_BRIDGE_NETFILTER
  448. new->nf_bridge = old->nf_bridge;
  449. nf_bridge_get(old->nf_bridge);
  450. #endif
  451. #endif
  452. #ifdef CONFIG_NET_SCHED
  453. #ifdef CONFIG_NET_CLS_ACT
  454. new->tc_verd = old->tc_verd;
  455. #endif
  456. new->tc_index = old->tc_index;
  457. #endif
  458. skb_copy_secmark(new, old);
  459. atomic_set(&new->users, 1);
  460. skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
  461. skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
  462. skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
  463. }
  464. /**
  465. * skb_copy - create private copy of an sk_buff
  466. * @skb: buffer to copy
  467. * @gfp_mask: allocation priority
  468. *
  469. * Make a copy of both an &sk_buff and its data. This is used when the
  470. * caller wishes to modify the data and needs a private copy of the
  471. * data to alter. Returns %NULL on failure or the pointer to the buffer
  472. * on success. The returned buffer has a reference count of 1.
  473. *
  474. * As by-product this function converts non-linear &sk_buff to linear
  475. * one, so that &sk_buff becomes completely private and caller is allowed
  476. * to modify all the data of returned buffer. This means that this
  477. * function is not recommended for use in circumstances when only
  478. * header is going to be modified. Use pskb_copy() instead.
  479. */
  480. struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
  481. {
  482. int headerlen = skb->data - skb->head;
  483. /*
  484. * Allocate the copy buffer
  485. */
  486. struct sk_buff *n = alloc_skb(skb->end - skb->head + skb->data_len,
  487. gfp_mask);
  488. if (!n)
  489. return NULL;
  490. /* Set the data pointer */
  491. skb_reserve(n, headerlen);
  492. /* Set the tail pointer and length */
  493. skb_put(n, skb->len);
  494. n->csum = skb->csum;
  495. n->ip_summed = skb->ip_summed;
  496. if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
  497. BUG();
  498. copy_skb_header(n, skb);
  499. return n;
  500. }
  501. /**
  502. * pskb_copy - create copy of an sk_buff with private head.
  503. * @skb: buffer to copy
  504. * @gfp_mask: allocation priority
  505. *
  506. * Make a copy of both an &sk_buff and part of its data, located
  507. * in header. Fragmented data remain shared. This is used when
  508. * the caller wishes to modify only header of &sk_buff and needs
  509. * private copy of the header to alter. Returns %NULL on failure
  510. * or the pointer to the buffer on success.
  511. * The returned buffer has a reference count of 1.
  512. */
  513. struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
  514. {
  515. /*
  516. * Allocate the copy buffer
  517. */
  518. struct sk_buff *n = alloc_skb(skb->end - skb->head, gfp_mask);
  519. if (!n)
  520. goto out;
  521. /* Set the data pointer */
  522. skb_reserve(n, skb->data - skb->head);
  523. /* Set the tail pointer and length */
  524. skb_put(n, skb_headlen(skb));
  525. /* Copy the bytes */
  526. memcpy(n->data, skb->data, n->len);
  527. n->csum = skb->csum;
  528. n->ip_summed = skb->ip_summed;
  529. n->truesize += skb->data_len;
  530. n->data_len = skb->data_len;
  531. n->len = skb->len;
  532. if (skb_shinfo(skb)->nr_frags) {
  533. int i;
  534. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  535. skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
  536. get_page(skb_shinfo(n)->frags[i].page);
  537. }
  538. skb_shinfo(n)->nr_frags = i;
  539. }
  540. if (skb_shinfo(skb)->frag_list) {
  541. skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
  542. skb_clone_fraglist(n);
  543. }
  544. copy_skb_header(n, skb);
  545. out:
  546. return n;
  547. }
  548. /**
  549. * pskb_expand_head - reallocate header of &sk_buff
  550. * @skb: buffer to reallocate
  551. * @nhead: room to add at head
  552. * @ntail: room to add at tail
  553. * @gfp_mask: allocation priority
  554. *
  555. * Expands (or creates identical copy, if &nhead and &ntail are zero)
  556. * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
  557. * reference count of 1. Returns zero in the case of success or error,
  558. * if expansion failed. In the last case, &sk_buff is not changed.
  559. *
  560. * All the pointers pointing into skb header may change and must be
  561. * reloaded after call to this function.
  562. */
  563. int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
  564. gfp_t gfp_mask)
  565. {
  566. int i;
  567. u8 *data;
  568. int size = nhead + (skb->end - skb->head) + ntail;
  569. long off;
  570. if (skb_shared(skb))
  571. BUG();
  572. size = SKB_DATA_ALIGN(size);
  573. data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
  574. if (!data)
  575. goto nodata;
  576. /* Copy only real data... and, alas, header. This should be
  577. * optimized for the cases when header is void. */
  578. memcpy(data + nhead, skb->head, skb->tail - skb->head);
  579. memcpy(data + size, skb->end, sizeof(struct skb_shared_info));
  580. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  581. get_page(skb_shinfo(skb)->frags[i].page);
  582. if (skb_shinfo(skb)->frag_list)
  583. skb_clone_fraglist(skb);
  584. skb_release_data(skb);
  585. off = (data + nhead) - skb->head;
  586. skb->head = data;
  587. skb->end = data + size;
  588. skb->data += off;
  589. skb->tail += off;
  590. skb->mac.raw += off;
  591. skb->h.raw += off;
  592. skb->nh.raw += off;
  593. skb->cloned = 0;
  594. skb->nohdr = 0;
  595. atomic_set(&skb_shinfo(skb)->dataref, 1);
  596. return 0;
  597. nodata:
  598. return -ENOMEM;
  599. }
  600. /* Make private copy of skb with writable head and some headroom */
  601. struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
  602. {
  603. struct sk_buff *skb2;
  604. int delta = headroom - skb_headroom(skb);
  605. if (delta <= 0)
  606. skb2 = pskb_copy(skb, GFP_ATOMIC);
  607. else {
  608. skb2 = skb_clone(skb, GFP_ATOMIC);
  609. if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
  610. GFP_ATOMIC)) {
  611. kfree_skb(skb2);
  612. skb2 = NULL;
  613. }
  614. }
  615. return skb2;
  616. }
  617. /**
  618. * skb_copy_expand - copy and expand sk_buff
  619. * @skb: buffer to copy
  620. * @newheadroom: new free bytes at head
  621. * @newtailroom: new free bytes at tail
  622. * @gfp_mask: allocation priority
  623. *
  624. * Make a copy of both an &sk_buff and its data and while doing so
  625. * allocate additional space.
  626. *
  627. * This is used when the caller wishes to modify the data and needs a
  628. * private copy of the data to alter as well as more space for new fields.
  629. * Returns %NULL on failure or the pointer to the buffer
  630. * on success. The returned buffer has a reference count of 1.
  631. *
  632. * You must pass %GFP_ATOMIC as the allocation priority if this function
  633. * is called from an interrupt.
  634. *
  635. * BUG ALERT: ip_summed is not copied. Why does this work? Is it used
  636. * only by netfilter in the cases when checksum is recalculated? --ANK
  637. */
  638. struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
  639. int newheadroom, int newtailroom,
  640. gfp_t gfp_mask)
  641. {
  642. /*
  643. * Allocate the copy buffer
  644. */
  645. struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
  646. gfp_mask);
  647. int head_copy_len, head_copy_off;
  648. if (!n)
  649. return NULL;
  650. skb_reserve(n, newheadroom);
  651. /* Set the tail pointer and length */
  652. skb_put(n, skb->len);
  653. head_copy_len = skb_headroom(skb);
  654. head_copy_off = 0;
  655. if (newheadroom <= head_copy_len)
  656. head_copy_len = newheadroom;
  657. else
  658. head_copy_off = newheadroom - head_copy_len;
  659. /* Copy the linear header and data. */
  660. if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
  661. skb->len + head_copy_len))
  662. BUG();
  663. copy_skb_header(n, skb);
  664. return n;
  665. }
  666. /**
  667. * skb_pad - zero pad the tail of an skb
  668. * @skb: buffer to pad
  669. * @pad: space to pad
  670. *
  671. * Ensure that a buffer is followed by a padding area that is zero
  672. * filled. Used by network drivers which may DMA or transfer data
  673. * beyond the buffer end onto the wire.
  674. *
  675. * May return error in out of memory cases. The skb is freed on error.
  676. */
  677. int skb_pad(struct sk_buff *skb, int pad)
  678. {
  679. int err;
  680. int ntail;
  681. /* If the skbuff is non linear tailroom is always zero.. */
  682. if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
  683. memset(skb->data+skb->len, 0, pad);
  684. return 0;
  685. }
  686. ntail = skb->data_len + pad - (skb->end - skb->tail);
  687. if (likely(skb_cloned(skb) || ntail > 0)) {
  688. err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
  689. if (unlikely(err))
  690. goto free_skb;
  691. }
  692. /* FIXME: The use of this function with non-linear skb's really needs
  693. * to be audited.
  694. */
  695. err = skb_linearize(skb);
  696. if (unlikely(err))
  697. goto free_skb;
  698. memset(skb->data + skb->len, 0, pad);
  699. return 0;
  700. free_skb:
  701. kfree_skb(skb);
  702. return err;
  703. }
  704. /* Trims skb to length len. It can change skb pointers.
  705. */
  706. int ___pskb_trim(struct sk_buff *skb, unsigned int len)
  707. {
  708. struct sk_buff **fragp;
  709. struct sk_buff *frag;
  710. int offset = skb_headlen(skb);
  711. int nfrags = skb_shinfo(skb)->nr_frags;
  712. int i;
  713. int err;
  714. if (skb_cloned(skb) &&
  715. unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
  716. return err;
  717. i = 0;
  718. if (offset >= len)
  719. goto drop_pages;
  720. for (; i < nfrags; i++) {
  721. int end = offset + skb_shinfo(skb)->frags[i].size;
  722. if (end < len) {
  723. offset = end;
  724. continue;
  725. }
  726. skb_shinfo(skb)->frags[i++].size = len - offset;
  727. drop_pages:
  728. skb_shinfo(skb)->nr_frags = i;
  729. for (; i < nfrags; i++)
  730. put_page(skb_shinfo(skb)->frags[i].page);
  731. if (skb_shinfo(skb)->frag_list)
  732. skb_drop_fraglist(skb);
  733. goto done;
  734. }
  735. for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
  736. fragp = &frag->next) {
  737. int end = offset + frag->len;
  738. if (skb_shared(frag)) {
  739. struct sk_buff *nfrag;
  740. nfrag = skb_clone(frag, GFP_ATOMIC);
  741. if (unlikely(!nfrag))
  742. return -ENOMEM;
  743. nfrag->next = frag->next;
  744. kfree_skb(frag);
  745. frag = nfrag;
  746. *fragp = frag;
  747. }
  748. if (end < len) {
  749. offset = end;
  750. continue;
  751. }
  752. if (end > len &&
  753. unlikely((err = pskb_trim(frag, len - offset))))
  754. return err;
  755. if (frag->next)
  756. skb_drop_list(&frag->next);
  757. break;
  758. }
  759. done:
  760. if (len > skb_headlen(skb)) {
  761. skb->data_len -= skb->len - len;
  762. skb->len = len;
  763. } else {
  764. skb->len = len;
  765. skb->data_len = 0;
  766. skb->tail = skb->data + len;
  767. }
  768. return 0;
  769. }
  770. /**
  771. * __pskb_pull_tail - advance tail of skb header
  772. * @skb: buffer to reallocate
  773. * @delta: number of bytes to advance tail
  774. *
  775. * The function makes a sense only on a fragmented &sk_buff,
  776. * it expands header moving its tail forward and copying necessary
  777. * data from fragmented part.
  778. *
  779. * &sk_buff MUST have reference count of 1.
  780. *
  781. * Returns %NULL (and &sk_buff does not change) if pull failed
  782. * or value of new tail of skb in the case of success.
  783. *
  784. * All the pointers pointing into skb header may change and must be
  785. * reloaded after call to this function.
  786. */
  787. /* Moves tail of skb head forward, copying data from fragmented part,
  788. * when it is necessary.
  789. * 1. It may fail due to malloc failure.
  790. * 2. It may change skb pointers.
  791. *
  792. * It is pretty complicated. Luckily, it is called only in exceptional cases.
  793. */
  794. unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
  795. {
  796. /* If skb has not enough free space at tail, get new one
  797. * plus 128 bytes for future expansions. If we have enough
  798. * room at tail, reallocate without expansion only if skb is cloned.
  799. */
  800. int i, k, eat = (skb->tail + delta) - skb->end;
  801. if (eat > 0 || skb_cloned(skb)) {
  802. if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
  803. GFP_ATOMIC))
  804. return NULL;
  805. }
  806. if (skb_copy_bits(skb, skb_headlen(skb), skb->tail, delta))
  807. BUG();
  808. /* Optimization: no fragments, no reasons to preestimate
  809. * size of pulled pages. Superb.
  810. */
  811. if (!skb_shinfo(skb)->frag_list)
  812. goto pull_pages;
  813. /* Estimate size of pulled pages. */
  814. eat = delta;
  815. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  816. if (skb_shinfo(skb)->frags[i].size >= eat)
  817. goto pull_pages;
  818. eat -= skb_shinfo(skb)->frags[i].size;
  819. }
  820. /* If we need update frag list, we are in troubles.
  821. * Certainly, it possible to add an offset to skb data,
  822. * but taking into account that pulling is expected to
  823. * be very rare operation, it is worth to fight against
  824. * further bloating skb head and crucify ourselves here instead.
  825. * Pure masohism, indeed. 8)8)
  826. */
  827. if (eat) {
  828. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  829. struct sk_buff *clone = NULL;
  830. struct sk_buff *insp = NULL;
  831. do {
  832. BUG_ON(!list);
  833. if (list->len <= eat) {
  834. /* Eaten as whole. */
  835. eat -= list->len;
  836. list = list->next;
  837. insp = list;
  838. } else {
  839. /* Eaten partially. */
  840. if (skb_shared(list)) {
  841. /* Sucks! We need to fork list. :-( */
  842. clone = skb_clone(list, GFP_ATOMIC);
  843. if (!clone)
  844. return NULL;
  845. insp = list->next;
  846. list = clone;
  847. } else {
  848. /* This may be pulled without
  849. * problems. */
  850. insp = list;
  851. }
  852. if (!pskb_pull(list, eat)) {
  853. if (clone)
  854. kfree_skb(clone);
  855. return NULL;
  856. }
  857. break;
  858. }
  859. } while (eat);
  860. /* Free pulled out fragments. */
  861. while ((list = skb_shinfo(skb)->frag_list) != insp) {
  862. skb_shinfo(skb)->frag_list = list->next;
  863. kfree_skb(list);
  864. }
  865. /* And insert new clone at head. */
  866. if (clone) {
  867. clone->next = list;
  868. skb_shinfo(skb)->frag_list = clone;
  869. }
  870. }
  871. /* Success! Now we may commit changes to skb data. */
  872. pull_pages:
  873. eat = delta;
  874. k = 0;
  875. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  876. if (skb_shinfo(skb)->frags[i].size <= eat) {
  877. put_page(skb_shinfo(skb)->frags[i].page);
  878. eat -= skb_shinfo(skb)->frags[i].size;
  879. } else {
  880. skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
  881. if (eat) {
  882. skb_shinfo(skb)->frags[k].page_offset += eat;
  883. skb_shinfo(skb)->frags[k].size -= eat;
  884. eat = 0;
  885. }
  886. k++;
  887. }
  888. }
  889. skb_shinfo(skb)->nr_frags = k;
  890. skb->tail += delta;
  891. skb->data_len -= delta;
  892. return skb->tail;
  893. }
  894. /* Copy some data bits from skb to kernel buffer. */
  895. int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
  896. {
  897. int i, copy;
  898. int start = skb_headlen(skb);
  899. if (offset > (int)skb->len - len)
  900. goto fault;
  901. /* Copy header. */
  902. if ((copy = start - offset) > 0) {
  903. if (copy > len)
  904. copy = len;
  905. memcpy(to, skb->data + offset, copy);
  906. if ((len -= copy) == 0)
  907. return 0;
  908. offset += copy;
  909. to += copy;
  910. }
  911. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  912. int end;
  913. BUG_TRAP(start <= offset + len);
  914. end = start + skb_shinfo(skb)->frags[i].size;
  915. if ((copy = end - offset) > 0) {
  916. u8 *vaddr;
  917. if (copy > len)
  918. copy = len;
  919. vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
  920. memcpy(to,
  921. vaddr + skb_shinfo(skb)->frags[i].page_offset+
  922. offset - start, copy);
  923. kunmap_skb_frag(vaddr);
  924. if ((len -= copy) == 0)
  925. return 0;
  926. offset += copy;
  927. to += copy;
  928. }
  929. start = end;
  930. }
  931. if (skb_shinfo(skb)->frag_list) {
  932. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  933. for (; list; list = list->next) {
  934. int end;
  935. BUG_TRAP(start <= offset + len);
  936. end = start + list->len;
  937. if ((copy = end - offset) > 0) {
  938. if (copy > len)
  939. copy = len;
  940. if (skb_copy_bits(list, offset - start,
  941. to, copy))
  942. goto fault;
  943. if ((len -= copy) == 0)
  944. return 0;
  945. offset += copy;
  946. to += copy;
  947. }
  948. start = end;
  949. }
  950. }
  951. if (!len)
  952. return 0;
  953. fault:
  954. return -EFAULT;
  955. }
  956. /**
  957. * skb_store_bits - store bits from kernel buffer to skb
  958. * @skb: destination buffer
  959. * @offset: offset in destination
  960. * @from: source buffer
  961. * @len: number of bytes to copy
  962. *
  963. * Copy the specified number of bytes from the source buffer to the
  964. * destination skb. This function handles all the messy bits of
  965. * traversing fragment lists and such.
  966. */
  967. int skb_store_bits(const struct sk_buff *skb, int offset, void *from, int len)
  968. {
  969. int i, copy;
  970. int start = skb_headlen(skb);
  971. if (offset > (int)skb->len - len)
  972. goto fault;
  973. if ((copy = start - offset) > 0) {
  974. if (copy > len)
  975. copy = len;
  976. memcpy(skb->data + offset, from, copy);
  977. if ((len -= copy) == 0)
  978. return 0;
  979. offset += copy;
  980. from += copy;
  981. }
  982. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  983. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  984. int end;
  985. BUG_TRAP(start <= offset + len);
  986. end = start + frag->size;
  987. if ((copy = end - offset) > 0) {
  988. u8 *vaddr;
  989. if (copy > len)
  990. copy = len;
  991. vaddr = kmap_skb_frag(frag);
  992. memcpy(vaddr + frag->page_offset + offset - start,
  993. from, copy);
  994. kunmap_skb_frag(vaddr);
  995. if ((len -= copy) == 0)
  996. return 0;
  997. offset += copy;
  998. from += copy;
  999. }
  1000. start = end;
  1001. }
  1002. if (skb_shinfo(skb)->frag_list) {
  1003. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1004. for (; list; list = list->next) {
  1005. int end;
  1006. BUG_TRAP(start <= offset + len);
  1007. end = start + list->len;
  1008. if ((copy = end - offset) > 0) {
  1009. if (copy > len)
  1010. copy = len;
  1011. if (skb_store_bits(list, offset - start,
  1012. from, copy))
  1013. goto fault;
  1014. if ((len -= copy) == 0)
  1015. return 0;
  1016. offset += copy;
  1017. from += copy;
  1018. }
  1019. start = end;
  1020. }
  1021. }
  1022. if (!len)
  1023. return 0;
  1024. fault:
  1025. return -EFAULT;
  1026. }
  1027. EXPORT_SYMBOL(skb_store_bits);
  1028. /* Checksum skb data. */
  1029. __wsum skb_checksum(const struct sk_buff *skb, int offset,
  1030. int len, __wsum csum)
  1031. {
  1032. int start = skb_headlen(skb);
  1033. int i, copy = start - offset;
  1034. int pos = 0;
  1035. /* Checksum header. */
  1036. if (copy > 0) {
  1037. if (copy > len)
  1038. copy = len;
  1039. csum = csum_partial(skb->data + offset, copy, csum);
  1040. if ((len -= copy) == 0)
  1041. return csum;
  1042. offset += copy;
  1043. pos = copy;
  1044. }
  1045. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1046. int end;
  1047. BUG_TRAP(start <= offset + len);
  1048. end = start + skb_shinfo(skb)->frags[i].size;
  1049. if ((copy = end - offset) > 0) {
  1050. __wsum csum2;
  1051. u8 *vaddr;
  1052. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1053. if (copy > len)
  1054. copy = len;
  1055. vaddr = kmap_skb_frag(frag);
  1056. csum2 = csum_partial(vaddr + frag->page_offset +
  1057. offset - start, copy, 0);
  1058. kunmap_skb_frag(vaddr);
  1059. csum = csum_block_add(csum, csum2, pos);
  1060. if (!(len -= copy))
  1061. return csum;
  1062. offset += copy;
  1063. pos += copy;
  1064. }
  1065. start = end;
  1066. }
  1067. if (skb_shinfo(skb)->frag_list) {
  1068. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1069. for (; list; list = list->next) {
  1070. int end;
  1071. BUG_TRAP(start <= offset + len);
  1072. end = start + list->len;
  1073. if ((copy = end - offset) > 0) {
  1074. __wsum csum2;
  1075. if (copy > len)
  1076. copy = len;
  1077. csum2 = skb_checksum(list, offset - start,
  1078. copy, 0);
  1079. csum = csum_block_add(csum, csum2, pos);
  1080. if ((len -= copy) == 0)
  1081. return csum;
  1082. offset += copy;
  1083. pos += copy;
  1084. }
  1085. start = end;
  1086. }
  1087. }
  1088. BUG_ON(len);
  1089. return csum;
  1090. }
  1091. /* Both of above in one bottle. */
  1092. __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
  1093. u8 *to, int len, __wsum csum)
  1094. {
  1095. int start = skb_headlen(skb);
  1096. int i, copy = start - offset;
  1097. int pos = 0;
  1098. /* Copy header. */
  1099. if (copy > 0) {
  1100. if (copy > len)
  1101. copy = len;
  1102. csum = csum_partial_copy_nocheck(skb->data + offset, to,
  1103. copy, csum);
  1104. if ((len -= copy) == 0)
  1105. return csum;
  1106. offset += copy;
  1107. to += copy;
  1108. pos = copy;
  1109. }
  1110. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1111. int end;
  1112. BUG_TRAP(start <= offset + len);
  1113. end = start + skb_shinfo(skb)->frags[i].size;
  1114. if ((copy = end - offset) > 0) {
  1115. __wsum csum2;
  1116. u8 *vaddr;
  1117. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1118. if (copy > len)
  1119. copy = len;
  1120. vaddr = kmap_skb_frag(frag);
  1121. csum2 = csum_partial_copy_nocheck(vaddr +
  1122. frag->page_offset +
  1123. offset - start, to,
  1124. copy, 0);
  1125. kunmap_skb_frag(vaddr);
  1126. csum = csum_block_add(csum, csum2, pos);
  1127. if (!(len -= copy))
  1128. return csum;
  1129. offset += copy;
  1130. to += copy;
  1131. pos += copy;
  1132. }
  1133. start = end;
  1134. }
  1135. if (skb_shinfo(skb)->frag_list) {
  1136. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1137. for (; list; list = list->next) {
  1138. __wsum csum2;
  1139. int end;
  1140. BUG_TRAP(start <= offset + len);
  1141. end = start + list->len;
  1142. if ((copy = end - offset) > 0) {
  1143. if (copy > len)
  1144. copy = len;
  1145. csum2 = skb_copy_and_csum_bits(list,
  1146. offset - start,
  1147. to, copy, 0);
  1148. csum = csum_block_add(csum, csum2, pos);
  1149. if ((len -= copy) == 0)
  1150. return csum;
  1151. offset += copy;
  1152. to += copy;
  1153. pos += copy;
  1154. }
  1155. start = end;
  1156. }
  1157. }
  1158. BUG_ON(len);
  1159. return csum;
  1160. }
  1161. void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
  1162. {
  1163. __wsum csum;
  1164. long csstart;
  1165. if (skb->ip_summed == CHECKSUM_PARTIAL)
  1166. csstart = skb->h.raw - skb->data;
  1167. else
  1168. csstart = skb_headlen(skb);
  1169. BUG_ON(csstart > skb_headlen(skb));
  1170. memcpy(to, skb->data, csstart);
  1171. csum = 0;
  1172. if (csstart != skb->len)
  1173. csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
  1174. skb->len - csstart, 0);
  1175. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  1176. long csstuff = csstart + skb->csum_offset;
  1177. *((__sum16 *)(to + csstuff)) = csum_fold(csum);
  1178. }
  1179. }
  1180. /**
  1181. * skb_dequeue - remove from the head of the queue
  1182. * @list: list to dequeue from
  1183. *
  1184. * Remove the head of the list. The list lock is taken so the function
  1185. * may be used safely with other locking list functions. The head item is
  1186. * returned or %NULL if the list is empty.
  1187. */
  1188. struct sk_buff *skb_dequeue(struct sk_buff_head *list)
  1189. {
  1190. unsigned long flags;
  1191. struct sk_buff *result;
  1192. spin_lock_irqsave(&list->lock, flags);
  1193. result = __skb_dequeue(list);
  1194. spin_unlock_irqrestore(&list->lock, flags);
  1195. return result;
  1196. }
  1197. /**
  1198. * skb_dequeue_tail - remove from the tail of the queue
  1199. * @list: list to dequeue from
  1200. *
  1201. * Remove the tail of the list. The list lock is taken so the function
  1202. * may be used safely with other locking list functions. The tail item is
  1203. * returned or %NULL if the list is empty.
  1204. */
  1205. struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
  1206. {
  1207. unsigned long flags;
  1208. struct sk_buff *result;
  1209. spin_lock_irqsave(&list->lock, flags);
  1210. result = __skb_dequeue_tail(list);
  1211. spin_unlock_irqrestore(&list->lock, flags);
  1212. return result;
  1213. }
  1214. /**
  1215. * skb_queue_purge - empty a list
  1216. * @list: list to empty
  1217. *
  1218. * Delete all buffers on an &sk_buff list. Each buffer is removed from
  1219. * the list and one reference dropped. This function takes the list
  1220. * lock and is atomic with respect to other list locking functions.
  1221. */
  1222. void skb_queue_purge(struct sk_buff_head *list)
  1223. {
  1224. struct sk_buff *skb;
  1225. while ((skb = skb_dequeue(list)) != NULL)
  1226. kfree_skb(skb);
  1227. }
  1228. /**
  1229. * skb_queue_head - queue a buffer at the list head
  1230. * @list: list to use
  1231. * @newsk: buffer to queue
  1232. *
  1233. * Queue a buffer at the start of the list. This function takes the
  1234. * list lock and can be used safely with other locking &sk_buff functions
  1235. * safely.
  1236. *
  1237. * A buffer cannot be placed on two lists at the same time.
  1238. */
  1239. void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
  1240. {
  1241. unsigned long flags;
  1242. spin_lock_irqsave(&list->lock, flags);
  1243. __skb_queue_head(list, newsk);
  1244. spin_unlock_irqrestore(&list->lock, flags);
  1245. }
  1246. /**
  1247. * skb_queue_tail - queue a buffer at the list tail
  1248. * @list: list to use
  1249. * @newsk: buffer to queue
  1250. *
  1251. * Queue a buffer at the tail of the list. This function takes the
  1252. * list lock and can be used safely with other locking &sk_buff functions
  1253. * safely.
  1254. *
  1255. * A buffer cannot be placed on two lists at the same time.
  1256. */
  1257. void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
  1258. {
  1259. unsigned long flags;
  1260. spin_lock_irqsave(&list->lock, flags);
  1261. __skb_queue_tail(list, newsk);
  1262. spin_unlock_irqrestore(&list->lock, flags);
  1263. }
  1264. /**
  1265. * skb_unlink - remove a buffer from a list
  1266. * @skb: buffer to remove
  1267. * @list: list to use
  1268. *
  1269. * Remove a packet from a list. The list locks are taken and this
  1270. * function is atomic with respect to other list locked calls
  1271. *
  1272. * You must know what list the SKB is on.
  1273. */
  1274. void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
  1275. {
  1276. unsigned long flags;
  1277. spin_lock_irqsave(&list->lock, flags);
  1278. __skb_unlink(skb, list);
  1279. spin_unlock_irqrestore(&list->lock, flags);
  1280. }
  1281. /**
  1282. * skb_append - append a buffer
  1283. * @old: buffer to insert after
  1284. * @newsk: buffer to insert
  1285. * @list: list to use
  1286. *
  1287. * Place a packet after a given packet in a list. The list locks are taken
  1288. * and this function is atomic with respect to other list locked calls.
  1289. * A buffer cannot be placed on two lists at the same time.
  1290. */
  1291. void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1292. {
  1293. unsigned long flags;
  1294. spin_lock_irqsave(&list->lock, flags);
  1295. __skb_append(old, newsk, list);
  1296. spin_unlock_irqrestore(&list->lock, flags);
  1297. }
  1298. /**
  1299. * skb_insert - insert a buffer
  1300. * @old: buffer to insert before
  1301. * @newsk: buffer to insert
  1302. * @list: list to use
  1303. *
  1304. * Place a packet before a given packet in a list. The list locks are
  1305. * taken and this function is atomic with respect to other list locked
  1306. * calls.
  1307. *
  1308. * A buffer cannot be placed on two lists at the same time.
  1309. */
  1310. void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1311. {
  1312. unsigned long flags;
  1313. spin_lock_irqsave(&list->lock, flags);
  1314. __skb_insert(newsk, old->prev, old, list);
  1315. spin_unlock_irqrestore(&list->lock, flags);
  1316. }
  1317. #if 0
  1318. /*
  1319. * Tune the memory allocator for a new MTU size.
  1320. */
  1321. void skb_add_mtu(int mtu)
  1322. {
  1323. /* Must match allocation in alloc_skb */
  1324. mtu = SKB_DATA_ALIGN(mtu) + sizeof(struct skb_shared_info);
  1325. kmem_add_cache_size(mtu);
  1326. }
  1327. #endif
  1328. static inline void skb_split_inside_header(struct sk_buff *skb,
  1329. struct sk_buff* skb1,
  1330. const u32 len, const int pos)
  1331. {
  1332. int i;
  1333. memcpy(skb_put(skb1, pos - len), skb->data + len, pos - len);
  1334. /* And move data appendix as is. */
  1335. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  1336. skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
  1337. skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
  1338. skb_shinfo(skb)->nr_frags = 0;
  1339. skb1->data_len = skb->data_len;
  1340. skb1->len += skb1->data_len;
  1341. skb->data_len = 0;
  1342. skb->len = len;
  1343. skb->tail = skb->data + len;
  1344. }
  1345. static inline void skb_split_no_header(struct sk_buff *skb,
  1346. struct sk_buff* skb1,
  1347. const u32 len, int pos)
  1348. {
  1349. int i, k = 0;
  1350. const int nfrags = skb_shinfo(skb)->nr_frags;
  1351. skb_shinfo(skb)->nr_frags = 0;
  1352. skb1->len = skb1->data_len = skb->len - len;
  1353. skb->len = len;
  1354. skb->data_len = len - pos;
  1355. for (i = 0; i < nfrags; i++) {
  1356. int size = skb_shinfo(skb)->frags[i].size;
  1357. if (pos + size > len) {
  1358. skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
  1359. if (pos < len) {
  1360. /* Split frag.
  1361. * We have two variants in this case:
  1362. * 1. Move all the frag to the second
  1363. * part, if it is possible. F.e.
  1364. * this approach is mandatory for TUX,
  1365. * where splitting is expensive.
  1366. * 2. Split is accurately. We make this.
  1367. */
  1368. get_page(skb_shinfo(skb)->frags[i].page);
  1369. skb_shinfo(skb1)->frags[0].page_offset += len - pos;
  1370. skb_shinfo(skb1)->frags[0].size -= len - pos;
  1371. skb_shinfo(skb)->frags[i].size = len - pos;
  1372. skb_shinfo(skb)->nr_frags++;
  1373. }
  1374. k++;
  1375. } else
  1376. skb_shinfo(skb)->nr_frags++;
  1377. pos += size;
  1378. }
  1379. skb_shinfo(skb1)->nr_frags = k;
  1380. }
  1381. /**
  1382. * skb_split - Split fragmented skb to two parts at length len.
  1383. * @skb: the buffer to split
  1384. * @skb1: the buffer to receive the second part
  1385. * @len: new length for skb
  1386. */
  1387. void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
  1388. {
  1389. int pos = skb_headlen(skb);
  1390. if (len < pos) /* Split line is inside header. */
  1391. skb_split_inside_header(skb, skb1, len, pos);
  1392. else /* Second chunk has no header, nothing to copy. */
  1393. skb_split_no_header(skb, skb1, len, pos);
  1394. }
  1395. /**
  1396. * skb_prepare_seq_read - Prepare a sequential read of skb data
  1397. * @skb: the buffer to read
  1398. * @from: lower offset of data to be read
  1399. * @to: upper offset of data to be read
  1400. * @st: state variable
  1401. *
  1402. * Initializes the specified state variable. Must be called before
  1403. * invoking skb_seq_read() for the first time.
  1404. */
  1405. void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
  1406. unsigned int to, struct skb_seq_state *st)
  1407. {
  1408. st->lower_offset = from;
  1409. st->upper_offset = to;
  1410. st->root_skb = st->cur_skb = skb;
  1411. st->frag_idx = st->stepped_offset = 0;
  1412. st->frag_data = NULL;
  1413. }
  1414. /**
  1415. * skb_seq_read - Sequentially read skb data
  1416. * @consumed: number of bytes consumed by the caller so far
  1417. * @data: destination pointer for data to be returned
  1418. * @st: state variable
  1419. *
  1420. * Reads a block of skb data at &consumed relative to the
  1421. * lower offset specified to skb_prepare_seq_read(). Assigns
  1422. * the head of the data block to &data and returns the length
  1423. * of the block or 0 if the end of the skb data or the upper
  1424. * offset has been reached.
  1425. *
  1426. * The caller is not required to consume all of the data
  1427. * returned, i.e. &consumed is typically set to the number
  1428. * of bytes already consumed and the next call to
  1429. * skb_seq_read() will return the remaining part of the block.
  1430. *
  1431. * Note: The size of each block of data returned can be arbitary,
  1432. * this limitation is the cost for zerocopy seqeuental
  1433. * reads of potentially non linear data.
  1434. *
  1435. * Note: Fragment lists within fragments are not implemented
  1436. * at the moment, state->root_skb could be replaced with
  1437. * a stack for this purpose.
  1438. */
  1439. unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
  1440. struct skb_seq_state *st)
  1441. {
  1442. unsigned int block_limit, abs_offset = consumed + st->lower_offset;
  1443. skb_frag_t *frag;
  1444. if (unlikely(abs_offset >= st->upper_offset))
  1445. return 0;
  1446. next_skb:
  1447. block_limit = skb_headlen(st->cur_skb);
  1448. if (abs_offset < block_limit) {
  1449. *data = st->cur_skb->data + abs_offset;
  1450. return block_limit - abs_offset;
  1451. }
  1452. if (st->frag_idx == 0 && !st->frag_data)
  1453. st->stepped_offset += skb_headlen(st->cur_skb);
  1454. while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
  1455. frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
  1456. block_limit = frag->size + st->stepped_offset;
  1457. if (abs_offset < block_limit) {
  1458. if (!st->frag_data)
  1459. st->frag_data = kmap_skb_frag(frag);
  1460. *data = (u8 *) st->frag_data + frag->page_offset +
  1461. (abs_offset - st->stepped_offset);
  1462. return block_limit - abs_offset;
  1463. }
  1464. if (st->frag_data) {
  1465. kunmap_skb_frag(st->frag_data);
  1466. st->frag_data = NULL;
  1467. }
  1468. st->frag_idx++;
  1469. st->stepped_offset += frag->size;
  1470. }
  1471. if (st->cur_skb->next) {
  1472. st->cur_skb = st->cur_skb->next;
  1473. st->frag_idx = 0;
  1474. goto next_skb;
  1475. } else if (st->root_skb == st->cur_skb &&
  1476. skb_shinfo(st->root_skb)->frag_list) {
  1477. st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
  1478. goto next_skb;
  1479. }
  1480. return 0;
  1481. }
  1482. /**
  1483. * skb_abort_seq_read - Abort a sequential read of skb data
  1484. * @st: state variable
  1485. *
  1486. * Must be called if skb_seq_read() was not called until it
  1487. * returned 0.
  1488. */
  1489. void skb_abort_seq_read(struct skb_seq_state *st)
  1490. {
  1491. if (st->frag_data)
  1492. kunmap_skb_frag(st->frag_data);
  1493. }
  1494. #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
  1495. static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
  1496. struct ts_config *conf,
  1497. struct ts_state *state)
  1498. {
  1499. return skb_seq_read(offset, text, TS_SKB_CB(state));
  1500. }
  1501. static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
  1502. {
  1503. skb_abort_seq_read(TS_SKB_CB(state));
  1504. }
  1505. /**
  1506. * skb_find_text - Find a text pattern in skb data
  1507. * @skb: the buffer to look in
  1508. * @from: search offset
  1509. * @to: search limit
  1510. * @config: textsearch configuration
  1511. * @state: uninitialized textsearch state variable
  1512. *
  1513. * Finds a pattern in the skb data according to the specified
  1514. * textsearch configuration. Use textsearch_next() to retrieve
  1515. * subsequent occurrences of the pattern. Returns the offset
  1516. * to the first occurrence or UINT_MAX if no match was found.
  1517. */
  1518. unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
  1519. unsigned int to, struct ts_config *config,
  1520. struct ts_state *state)
  1521. {
  1522. unsigned int ret;
  1523. config->get_next_block = skb_ts_get_next_block;
  1524. config->finish = skb_ts_finish;
  1525. skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
  1526. ret = textsearch_find(config, state);
  1527. return (ret <= to - from ? ret : UINT_MAX);
  1528. }
  1529. /**
  1530. * skb_append_datato_frags: - append the user data to a skb
  1531. * @sk: sock structure
  1532. * @skb: skb structure to be appened with user data.
  1533. * @getfrag: call back function to be used for getting the user data
  1534. * @from: pointer to user message iov
  1535. * @length: length of the iov message
  1536. *
  1537. * Description: This procedure append the user data in the fragment part
  1538. * of the skb if any page alloc fails user this procedure returns -ENOMEM
  1539. */
  1540. int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
  1541. int (*getfrag)(void *from, char *to, int offset,
  1542. int len, int odd, struct sk_buff *skb),
  1543. void *from, int length)
  1544. {
  1545. int frg_cnt = 0;
  1546. skb_frag_t *frag = NULL;
  1547. struct page *page = NULL;
  1548. int copy, left;
  1549. int offset = 0;
  1550. int ret;
  1551. do {
  1552. /* Return error if we don't have space for new frag */
  1553. frg_cnt = skb_shinfo(skb)->nr_frags;
  1554. if (frg_cnt >= MAX_SKB_FRAGS)
  1555. return -EFAULT;
  1556. /* allocate a new page for next frag */
  1557. page = alloc_pages(sk->sk_allocation, 0);
  1558. /* If alloc_page fails just return failure and caller will
  1559. * free previous allocated pages by doing kfree_skb()
  1560. */
  1561. if (page == NULL)
  1562. return -ENOMEM;
  1563. /* initialize the next frag */
  1564. sk->sk_sndmsg_page = page;
  1565. sk->sk_sndmsg_off = 0;
  1566. skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
  1567. skb->truesize += PAGE_SIZE;
  1568. atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
  1569. /* get the new initialized frag */
  1570. frg_cnt = skb_shinfo(skb)->nr_frags;
  1571. frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
  1572. /* copy the user data to page */
  1573. left = PAGE_SIZE - frag->page_offset;
  1574. copy = (length > left)? left : length;
  1575. ret = getfrag(from, (page_address(frag->page) +
  1576. frag->page_offset + frag->size),
  1577. offset, copy, 0, skb);
  1578. if (ret < 0)
  1579. return -EFAULT;
  1580. /* copy was successful so update the size parameters */
  1581. sk->sk_sndmsg_off += copy;
  1582. frag->size += copy;
  1583. skb->len += copy;
  1584. skb->data_len += copy;
  1585. offset += copy;
  1586. length -= copy;
  1587. } while (length > 0);
  1588. return 0;
  1589. }
  1590. /**
  1591. * skb_pull_rcsum - pull skb and update receive checksum
  1592. * @skb: buffer to update
  1593. * @start: start of data before pull
  1594. * @len: length of data pulled
  1595. *
  1596. * This function performs an skb_pull on the packet and updates
  1597. * update the CHECKSUM_COMPLETE checksum. It should be used on
  1598. * receive path processing instead of skb_pull unless you know
  1599. * that the checksum difference is zero (e.g., a valid IP header)
  1600. * or you are setting ip_summed to CHECKSUM_NONE.
  1601. */
  1602. unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
  1603. {
  1604. BUG_ON(len > skb->len);
  1605. skb->len -= len;
  1606. BUG_ON(skb->len < skb->data_len);
  1607. skb_postpull_rcsum(skb, skb->data, len);
  1608. return skb->data += len;
  1609. }
  1610. EXPORT_SYMBOL_GPL(skb_pull_rcsum);
  1611. /**
  1612. * skb_segment - Perform protocol segmentation on skb.
  1613. * @skb: buffer to segment
  1614. * @features: features for the output path (see dev->features)
  1615. *
  1616. * This function performs segmentation on the given skb. It returns
  1617. * the segment at the given position. It returns NULL if there are
  1618. * no more segments to generate, or when an error is encountered.
  1619. */
  1620. struct sk_buff *skb_segment(struct sk_buff *skb, int features)
  1621. {
  1622. struct sk_buff *segs = NULL;
  1623. struct sk_buff *tail = NULL;
  1624. unsigned int mss = skb_shinfo(skb)->gso_size;
  1625. unsigned int doffset = skb->data - skb->mac.raw;
  1626. unsigned int offset = doffset;
  1627. unsigned int headroom;
  1628. unsigned int len;
  1629. int sg = features & NETIF_F_SG;
  1630. int nfrags = skb_shinfo(skb)->nr_frags;
  1631. int err = -ENOMEM;
  1632. int i = 0;
  1633. int pos;
  1634. __skb_push(skb, doffset);
  1635. headroom = skb_headroom(skb);
  1636. pos = skb_headlen(skb);
  1637. do {
  1638. struct sk_buff *nskb;
  1639. skb_frag_t *frag;
  1640. int hsize;
  1641. int k;
  1642. int size;
  1643. len = skb->len - offset;
  1644. if (len > mss)
  1645. len = mss;
  1646. hsize = skb_headlen(skb) - offset;
  1647. if (hsize < 0)
  1648. hsize = 0;
  1649. if (hsize > len || !sg)
  1650. hsize = len;
  1651. nskb = alloc_skb(hsize + doffset + headroom, GFP_ATOMIC);
  1652. if (unlikely(!nskb))
  1653. goto err;
  1654. if (segs)
  1655. tail->next = nskb;
  1656. else
  1657. segs = nskb;
  1658. tail = nskb;
  1659. nskb->dev = skb->dev;
  1660. nskb->priority = skb->priority;
  1661. nskb->protocol = skb->protocol;
  1662. nskb->dst = dst_clone(skb->dst);
  1663. memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
  1664. nskb->pkt_type = skb->pkt_type;
  1665. nskb->mac_len = skb->mac_len;
  1666. skb_reserve(nskb, headroom);
  1667. nskb->mac.raw = nskb->data;
  1668. nskb->nh.raw = nskb->data + skb->mac_len;
  1669. nskb->h.raw = nskb->nh.raw + (skb->h.raw - skb->nh.raw);
  1670. memcpy(skb_put(nskb, doffset), skb->data, doffset);
  1671. if (!sg) {
  1672. nskb->csum = skb_copy_and_csum_bits(skb, offset,
  1673. skb_put(nskb, len),
  1674. len, 0);
  1675. continue;
  1676. }
  1677. frag = skb_shinfo(nskb)->frags;
  1678. k = 0;
  1679. nskb->ip_summed = CHECKSUM_PARTIAL;
  1680. nskb->csum = skb->csum;
  1681. memcpy(skb_put(nskb, hsize), skb->data + offset, hsize);
  1682. while (pos < offset + len) {
  1683. BUG_ON(i >= nfrags);
  1684. *frag = skb_shinfo(skb)->frags[i];
  1685. get_page(frag->page);
  1686. size = frag->size;
  1687. if (pos < offset) {
  1688. frag->page_offset += offset - pos;
  1689. frag->size -= offset - pos;
  1690. }
  1691. k++;
  1692. if (pos + size <= offset + len) {
  1693. i++;
  1694. pos += size;
  1695. } else {
  1696. frag->size -= pos + size - (offset + len);
  1697. break;
  1698. }
  1699. frag++;
  1700. }
  1701. skb_shinfo(nskb)->nr_frags = k;
  1702. nskb->data_len = len - hsize;
  1703. nskb->len += nskb->data_len;
  1704. nskb->truesize += nskb->data_len;
  1705. } while ((offset += len) < skb->len);
  1706. return segs;
  1707. err:
  1708. while ((skb = segs)) {
  1709. segs = skb->next;
  1710. kfree_skb(skb);
  1711. }
  1712. return ERR_PTR(err);
  1713. }
  1714. EXPORT_SYMBOL_GPL(skb_segment);
  1715. void __init skb_init(void)
  1716. {
  1717. skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
  1718. sizeof(struct sk_buff),
  1719. 0,
  1720. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  1721. NULL, NULL);
  1722. skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
  1723. (2*sizeof(struct sk_buff)) +
  1724. sizeof(atomic_t),
  1725. 0,
  1726. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  1727. NULL, NULL);
  1728. }
  1729. EXPORT_SYMBOL(___pskb_trim);
  1730. EXPORT_SYMBOL(__kfree_skb);
  1731. EXPORT_SYMBOL(kfree_skb);
  1732. EXPORT_SYMBOL(__pskb_pull_tail);
  1733. EXPORT_SYMBOL(__alloc_skb);
  1734. EXPORT_SYMBOL(__netdev_alloc_skb);
  1735. EXPORT_SYMBOL(pskb_copy);
  1736. EXPORT_SYMBOL(pskb_expand_head);
  1737. EXPORT_SYMBOL(skb_checksum);
  1738. EXPORT_SYMBOL(skb_clone);
  1739. EXPORT_SYMBOL(skb_clone_fraglist);
  1740. EXPORT_SYMBOL(skb_copy);
  1741. EXPORT_SYMBOL(skb_copy_and_csum_bits);
  1742. EXPORT_SYMBOL(skb_copy_and_csum_dev);
  1743. EXPORT_SYMBOL(skb_copy_bits);
  1744. EXPORT_SYMBOL(skb_copy_expand);
  1745. EXPORT_SYMBOL(skb_over_panic);
  1746. EXPORT_SYMBOL(skb_pad);
  1747. EXPORT_SYMBOL(skb_realloc_headroom);
  1748. EXPORT_SYMBOL(skb_under_panic);
  1749. EXPORT_SYMBOL(skb_dequeue);
  1750. EXPORT_SYMBOL(skb_dequeue_tail);
  1751. EXPORT_SYMBOL(skb_insert);
  1752. EXPORT_SYMBOL(skb_queue_purge);
  1753. EXPORT_SYMBOL(skb_queue_head);
  1754. EXPORT_SYMBOL(skb_queue_tail);
  1755. EXPORT_SYMBOL(skb_unlink);
  1756. EXPORT_SYMBOL(skb_append);
  1757. EXPORT_SYMBOL(skb_split);
  1758. EXPORT_SYMBOL(skb_prepare_seq_read);
  1759. EXPORT_SYMBOL(skb_seq_read);
  1760. EXPORT_SYMBOL(skb_abort_seq_read);
  1761. EXPORT_SYMBOL(skb_find_text);
  1762. EXPORT_SYMBOL(skb_append_datato_frags);