xfrm_policy.c 58 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492
  1. /*
  2. * xfrm_policy.c
  3. *
  4. * Changes:
  5. * Mitsuru KANDA @USAGI
  6. * Kazunori MIYAZAWA @USAGI
  7. * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
  8. * IPv6 support
  9. * Kazunori MIYAZAWA @USAGI
  10. * YOSHIFUJI Hideaki
  11. * Split up af-specific portion
  12. * Derek Atkins <derek@ihtfp.com> Add the post_input processor
  13. *
  14. */
  15. #include <linux/slab.h>
  16. #include <linux/kmod.h>
  17. #include <linux/list.h>
  18. #include <linux/spinlock.h>
  19. #include <linux/workqueue.h>
  20. #include <linux/notifier.h>
  21. #include <linux/netdevice.h>
  22. #include <linux/netfilter.h>
  23. #include <linux/module.h>
  24. #include <linux/cache.h>
  25. #include <net/xfrm.h>
  26. #include <net/ip.h>
  27. #include <linux/audit.h>
  28. #include "xfrm_hash.h"
  29. DEFINE_MUTEX(xfrm_cfg_mutex);
  30. EXPORT_SYMBOL(xfrm_cfg_mutex);
  31. static DEFINE_RWLOCK(xfrm_policy_lock);
  32. unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2];
  33. EXPORT_SYMBOL(xfrm_policy_count);
  34. static DEFINE_RWLOCK(xfrm_policy_afinfo_lock);
  35. static struct xfrm_policy_afinfo *xfrm_policy_afinfo[NPROTO];
  36. static struct kmem_cache *xfrm_dst_cache __read_mostly;
  37. static struct work_struct xfrm_policy_gc_work;
  38. static HLIST_HEAD(xfrm_policy_gc_list);
  39. static DEFINE_SPINLOCK(xfrm_policy_gc_lock);
  40. static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family);
  41. static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo);
  42. static struct xfrm_policy_afinfo *xfrm_policy_lock_afinfo(unsigned int family);
  43. static void xfrm_policy_unlock_afinfo(struct xfrm_policy_afinfo *afinfo);
  44. static inline int
  45. __xfrm4_selector_match(struct xfrm_selector *sel, struct flowi *fl)
  46. {
  47. return addr_match(&fl->fl4_dst, &sel->daddr, sel->prefixlen_d) &&
  48. addr_match(&fl->fl4_src, &sel->saddr, sel->prefixlen_s) &&
  49. !((xfrm_flowi_dport(fl) ^ sel->dport) & sel->dport_mask) &&
  50. !((xfrm_flowi_sport(fl) ^ sel->sport) & sel->sport_mask) &&
  51. (fl->proto == sel->proto || !sel->proto) &&
  52. (fl->oif == sel->ifindex || !sel->ifindex);
  53. }
  54. static inline int
  55. __xfrm6_selector_match(struct xfrm_selector *sel, struct flowi *fl)
  56. {
  57. return addr_match(&fl->fl6_dst, &sel->daddr, sel->prefixlen_d) &&
  58. addr_match(&fl->fl6_src, &sel->saddr, sel->prefixlen_s) &&
  59. !((xfrm_flowi_dport(fl) ^ sel->dport) & sel->dport_mask) &&
  60. !((xfrm_flowi_sport(fl) ^ sel->sport) & sel->sport_mask) &&
  61. (fl->proto == sel->proto || !sel->proto) &&
  62. (fl->oif == sel->ifindex || !sel->ifindex);
  63. }
  64. int xfrm_selector_match(struct xfrm_selector *sel, struct flowi *fl,
  65. unsigned short family)
  66. {
  67. switch (family) {
  68. case AF_INET:
  69. return __xfrm4_selector_match(sel, fl);
  70. case AF_INET6:
  71. return __xfrm6_selector_match(sel, fl);
  72. }
  73. return 0;
  74. }
  75. int xfrm_register_type(struct xfrm_type *type, unsigned short family)
  76. {
  77. struct xfrm_policy_afinfo *afinfo = xfrm_policy_lock_afinfo(family);
  78. struct xfrm_type **typemap;
  79. int err = 0;
  80. if (unlikely(afinfo == NULL))
  81. return -EAFNOSUPPORT;
  82. typemap = afinfo->type_map;
  83. if (likely(typemap[type->proto] == NULL))
  84. typemap[type->proto] = type;
  85. else
  86. err = -EEXIST;
  87. xfrm_policy_unlock_afinfo(afinfo);
  88. return err;
  89. }
  90. EXPORT_SYMBOL(xfrm_register_type);
  91. int xfrm_unregister_type(struct xfrm_type *type, unsigned short family)
  92. {
  93. struct xfrm_policy_afinfo *afinfo = xfrm_policy_lock_afinfo(family);
  94. struct xfrm_type **typemap;
  95. int err = 0;
  96. if (unlikely(afinfo == NULL))
  97. return -EAFNOSUPPORT;
  98. typemap = afinfo->type_map;
  99. if (unlikely(typemap[type->proto] != type))
  100. err = -ENOENT;
  101. else
  102. typemap[type->proto] = NULL;
  103. xfrm_policy_unlock_afinfo(afinfo);
  104. return err;
  105. }
  106. EXPORT_SYMBOL(xfrm_unregister_type);
  107. struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
  108. {
  109. struct xfrm_policy_afinfo *afinfo;
  110. struct xfrm_type **typemap;
  111. struct xfrm_type *type;
  112. int modload_attempted = 0;
  113. retry:
  114. afinfo = xfrm_policy_get_afinfo(family);
  115. if (unlikely(afinfo == NULL))
  116. return NULL;
  117. typemap = afinfo->type_map;
  118. type = typemap[proto];
  119. if (unlikely(type && !try_module_get(type->owner)))
  120. type = NULL;
  121. if (!type && !modload_attempted) {
  122. xfrm_policy_put_afinfo(afinfo);
  123. request_module("xfrm-type-%d-%d",
  124. (int) family, (int) proto);
  125. modload_attempted = 1;
  126. goto retry;
  127. }
  128. xfrm_policy_put_afinfo(afinfo);
  129. return type;
  130. }
  131. int xfrm_dst_lookup(struct xfrm_dst **dst, struct flowi *fl,
  132. unsigned short family)
  133. {
  134. struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
  135. int err = 0;
  136. if (unlikely(afinfo == NULL))
  137. return -EAFNOSUPPORT;
  138. if (likely(afinfo->dst_lookup != NULL))
  139. err = afinfo->dst_lookup(dst, fl);
  140. else
  141. err = -EINVAL;
  142. xfrm_policy_put_afinfo(afinfo);
  143. return err;
  144. }
  145. EXPORT_SYMBOL(xfrm_dst_lookup);
  146. void xfrm_put_type(struct xfrm_type *type)
  147. {
  148. module_put(type->owner);
  149. }
  150. int xfrm_register_mode(struct xfrm_mode *mode, int family)
  151. {
  152. struct xfrm_policy_afinfo *afinfo;
  153. struct xfrm_mode **modemap;
  154. int err;
  155. if (unlikely(mode->encap >= XFRM_MODE_MAX))
  156. return -EINVAL;
  157. afinfo = xfrm_policy_lock_afinfo(family);
  158. if (unlikely(afinfo == NULL))
  159. return -EAFNOSUPPORT;
  160. err = -EEXIST;
  161. modemap = afinfo->mode_map;
  162. if (likely(modemap[mode->encap] == NULL)) {
  163. modemap[mode->encap] = mode;
  164. err = 0;
  165. }
  166. xfrm_policy_unlock_afinfo(afinfo);
  167. return err;
  168. }
  169. EXPORT_SYMBOL(xfrm_register_mode);
  170. int xfrm_unregister_mode(struct xfrm_mode *mode, int family)
  171. {
  172. struct xfrm_policy_afinfo *afinfo;
  173. struct xfrm_mode **modemap;
  174. int err;
  175. if (unlikely(mode->encap >= XFRM_MODE_MAX))
  176. return -EINVAL;
  177. afinfo = xfrm_policy_lock_afinfo(family);
  178. if (unlikely(afinfo == NULL))
  179. return -EAFNOSUPPORT;
  180. err = -ENOENT;
  181. modemap = afinfo->mode_map;
  182. if (likely(modemap[mode->encap] == mode)) {
  183. modemap[mode->encap] = NULL;
  184. err = 0;
  185. }
  186. xfrm_policy_unlock_afinfo(afinfo);
  187. return err;
  188. }
  189. EXPORT_SYMBOL(xfrm_unregister_mode);
  190. struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
  191. {
  192. struct xfrm_policy_afinfo *afinfo;
  193. struct xfrm_mode *mode;
  194. int modload_attempted = 0;
  195. if (unlikely(encap >= XFRM_MODE_MAX))
  196. return NULL;
  197. retry:
  198. afinfo = xfrm_policy_get_afinfo(family);
  199. if (unlikely(afinfo == NULL))
  200. return NULL;
  201. mode = afinfo->mode_map[encap];
  202. if (unlikely(mode && !try_module_get(mode->owner)))
  203. mode = NULL;
  204. if (!mode && !modload_attempted) {
  205. xfrm_policy_put_afinfo(afinfo);
  206. request_module("xfrm-mode-%d-%d", family, encap);
  207. modload_attempted = 1;
  208. goto retry;
  209. }
  210. xfrm_policy_put_afinfo(afinfo);
  211. return mode;
  212. }
  213. void xfrm_put_mode(struct xfrm_mode *mode)
  214. {
  215. module_put(mode->owner);
  216. }
  217. static inline unsigned long make_jiffies(long secs)
  218. {
  219. if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
  220. return MAX_SCHEDULE_TIMEOUT-1;
  221. else
  222. return secs*HZ;
  223. }
  224. static void xfrm_policy_timer(unsigned long data)
  225. {
  226. struct xfrm_policy *xp = (struct xfrm_policy*)data;
  227. unsigned long now = (unsigned long)xtime.tv_sec;
  228. long next = LONG_MAX;
  229. int warn = 0;
  230. int dir;
  231. read_lock(&xp->lock);
  232. if (xp->dead)
  233. goto out;
  234. dir = xfrm_policy_id2dir(xp->index);
  235. if (xp->lft.hard_add_expires_seconds) {
  236. long tmo = xp->lft.hard_add_expires_seconds +
  237. xp->curlft.add_time - now;
  238. if (tmo <= 0)
  239. goto expired;
  240. if (tmo < next)
  241. next = tmo;
  242. }
  243. if (xp->lft.hard_use_expires_seconds) {
  244. long tmo = xp->lft.hard_use_expires_seconds +
  245. (xp->curlft.use_time ? : xp->curlft.add_time) - now;
  246. if (tmo <= 0)
  247. goto expired;
  248. if (tmo < next)
  249. next = tmo;
  250. }
  251. if (xp->lft.soft_add_expires_seconds) {
  252. long tmo = xp->lft.soft_add_expires_seconds +
  253. xp->curlft.add_time - now;
  254. if (tmo <= 0) {
  255. warn = 1;
  256. tmo = XFRM_KM_TIMEOUT;
  257. }
  258. if (tmo < next)
  259. next = tmo;
  260. }
  261. if (xp->lft.soft_use_expires_seconds) {
  262. long tmo = xp->lft.soft_use_expires_seconds +
  263. (xp->curlft.use_time ? : xp->curlft.add_time) - now;
  264. if (tmo <= 0) {
  265. warn = 1;
  266. tmo = XFRM_KM_TIMEOUT;
  267. }
  268. if (tmo < next)
  269. next = tmo;
  270. }
  271. if (warn)
  272. km_policy_expired(xp, dir, 0, 0);
  273. if (next != LONG_MAX &&
  274. !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
  275. xfrm_pol_hold(xp);
  276. out:
  277. read_unlock(&xp->lock);
  278. xfrm_pol_put(xp);
  279. return;
  280. expired:
  281. read_unlock(&xp->lock);
  282. if (!xfrm_policy_delete(xp, dir))
  283. km_policy_expired(xp, dir, 1, 0);
  284. xfrm_pol_put(xp);
  285. }
  286. /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
  287. * SPD calls.
  288. */
  289. struct xfrm_policy *xfrm_policy_alloc(gfp_t gfp)
  290. {
  291. struct xfrm_policy *policy;
  292. policy = kzalloc(sizeof(struct xfrm_policy), gfp);
  293. if (policy) {
  294. INIT_HLIST_NODE(&policy->bydst);
  295. INIT_HLIST_NODE(&policy->byidx);
  296. rwlock_init(&policy->lock);
  297. atomic_set(&policy->refcnt, 1);
  298. init_timer(&policy->timer);
  299. policy->timer.data = (unsigned long)policy;
  300. policy->timer.function = xfrm_policy_timer;
  301. }
  302. return policy;
  303. }
  304. EXPORT_SYMBOL(xfrm_policy_alloc);
  305. /* Destroy xfrm_policy: descendant resources must be released to this moment. */
  306. void __xfrm_policy_destroy(struct xfrm_policy *policy)
  307. {
  308. BUG_ON(!policy->dead);
  309. BUG_ON(policy->bundles);
  310. if (del_timer(&policy->timer))
  311. BUG();
  312. security_xfrm_policy_free(policy);
  313. kfree(policy);
  314. }
  315. EXPORT_SYMBOL(__xfrm_policy_destroy);
  316. static void xfrm_policy_gc_kill(struct xfrm_policy *policy)
  317. {
  318. struct dst_entry *dst;
  319. while ((dst = policy->bundles) != NULL) {
  320. policy->bundles = dst->next;
  321. dst_free(dst);
  322. }
  323. if (del_timer(&policy->timer))
  324. atomic_dec(&policy->refcnt);
  325. if (atomic_read(&policy->refcnt) > 1)
  326. flow_cache_flush();
  327. xfrm_pol_put(policy);
  328. }
  329. static void xfrm_policy_gc_task(struct work_struct *work)
  330. {
  331. struct xfrm_policy *policy;
  332. struct hlist_node *entry, *tmp;
  333. struct hlist_head gc_list;
  334. spin_lock_bh(&xfrm_policy_gc_lock);
  335. gc_list.first = xfrm_policy_gc_list.first;
  336. INIT_HLIST_HEAD(&xfrm_policy_gc_list);
  337. spin_unlock_bh(&xfrm_policy_gc_lock);
  338. hlist_for_each_entry_safe(policy, entry, tmp, &gc_list, bydst)
  339. xfrm_policy_gc_kill(policy);
  340. }
  341. /* Rule must be locked. Release descentant resources, announce
  342. * entry dead. The rule must be unlinked from lists to the moment.
  343. */
  344. static void xfrm_policy_kill(struct xfrm_policy *policy)
  345. {
  346. int dead;
  347. write_lock_bh(&policy->lock);
  348. dead = policy->dead;
  349. policy->dead = 1;
  350. write_unlock_bh(&policy->lock);
  351. if (unlikely(dead)) {
  352. WARN_ON(1);
  353. return;
  354. }
  355. spin_lock(&xfrm_policy_gc_lock);
  356. hlist_add_head(&policy->bydst, &xfrm_policy_gc_list);
  357. spin_unlock(&xfrm_policy_gc_lock);
  358. schedule_work(&xfrm_policy_gc_work);
  359. }
  360. struct xfrm_policy_hash {
  361. struct hlist_head *table;
  362. unsigned int hmask;
  363. };
  364. static struct hlist_head xfrm_policy_inexact[XFRM_POLICY_MAX*2];
  365. static struct xfrm_policy_hash xfrm_policy_bydst[XFRM_POLICY_MAX*2] __read_mostly;
  366. static struct hlist_head *xfrm_policy_byidx __read_mostly;
  367. static unsigned int xfrm_idx_hmask __read_mostly;
  368. static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
  369. static inline unsigned int idx_hash(u32 index)
  370. {
  371. return __idx_hash(index, xfrm_idx_hmask);
  372. }
  373. static struct hlist_head *policy_hash_bysel(struct xfrm_selector *sel, unsigned short family, int dir)
  374. {
  375. unsigned int hmask = xfrm_policy_bydst[dir].hmask;
  376. unsigned int hash = __sel_hash(sel, family, hmask);
  377. return (hash == hmask + 1 ?
  378. &xfrm_policy_inexact[dir] :
  379. xfrm_policy_bydst[dir].table + hash);
  380. }
  381. static struct hlist_head *policy_hash_direct(xfrm_address_t *daddr, xfrm_address_t *saddr, unsigned short family, int dir)
  382. {
  383. unsigned int hmask = xfrm_policy_bydst[dir].hmask;
  384. unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
  385. return xfrm_policy_bydst[dir].table + hash;
  386. }
  387. static void xfrm_dst_hash_transfer(struct hlist_head *list,
  388. struct hlist_head *ndsttable,
  389. unsigned int nhashmask)
  390. {
  391. struct hlist_node *entry, *tmp;
  392. struct xfrm_policy *pol;
  393. hlist_for_each_entry_safe(pol, entry, tmp, list, bydst) {
  394. unsigned int h;
  395. h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
  396. pol->family, nhashmask);
  397. hlist_add_head(&pol->bydst, ndsttable+h);
  398. }
  399. }
  400. static void xfrm_idx_hash_transfer(struct hlist_head *list,
  401. struct hlist_head *nidxtable,
  402. unsigned int nhashmask)
  403. {
  404. struct hlist_node *entry, *tmp;
  405. struct xfrm_policy *pol;
  406. hlist_for_each_entry_safe(pol, entry, tmp, list, byidx) {
  407. unsigned int h;
  408. h = __idx_hash(pol->index, nhashmask);
  409. hlist_add_head(&pol->byidx, nidxtable+h);
  410. }
  411. }
  412. static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
  413. {
  414. return ((old_hmask + 1) << 1) - 1;
  415. }
  416. static void xfrm_bydst_resize(int dir)
  417. {
  418. unsigned int hmask = xfrm_policy_bydst[dir].hmask;
  419. unsigned int nhashmask = xfrm_new_hash_mask(hmask);
  420. unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
  421. struct hlist_head *odst = xfrm_policy_bydst[dir].table;
  422. struct hlist_head *ndst = xfrm_hash_alloc(nsize);
  423. int i;
  424. if (!ndst)
  425. return;
  426. write_lock_bh(&xfrm_policy_lock);
  427. for (i = hmask; i >= 0; i--)
  428. xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
  429. xfrm_policy_bydst[dir].table = ndst;
  430. xfrm_policy_bydst[dir].hmask = nhashmask;
  431. write_unlock_bh(&xfrm_policy_lock);
  432. xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
  433. }
  434. static void xfrm_byidx_resize(int total)
  435. {
  436. unsigned int hmask = xfrm_idx_hmask;
  437. unsigned int nhashmask = xfrm_new_hash_mask(hmask);
  438. unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
  439. struct hlist_head *oidx = xfrm_policy_byidx;
  440. struct hlist_head *nidx = xfrm_hash_alloc(nsize);
  441. int i;
  442. if (!nidx)
  443. return;
  444. write_lock_bh(&xfrm_policy_lock);
  445. for (i = hmask; i >= 0; i--)
  446. xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
  447. xfrm_policy_byidx = nidx;
  448. xfrm_idx_hmask = nhashmask;
  449. write_unlock_bh(&xfrm_policy_lock);
  450. xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
  451. }
  452. static inline int xfrm_bydst_should_resize(int dir, int *total)
  453. {
  454. unsigned int cnt = xfrm_policy_count[dir];
  455. unsigned int hmask = xfrm_policy_bydst[dir].hmask;
  456. if (total)
  457. *total += cnt;
  458. if ((hmask + 1) < xfrm_policy_hashmax &&
  459. cnt > hmask)
  460. return 1;
  461. return 0;
  462. }
  463. static inline int xfrm_byidx_should_resize(int total)
  464. {
  465. unsigned int hmask = xfrm_idx_hmask;
  466. if ((hmask + 1) < xfrm_policy_hashmax &&
  467. total > hmask)
  468. return 1;
  469. return 0;
  470. }
  471. static DEFINE_MUTEX(hash_resize_mutex);
  472. static void xfrm_hash_resize(struct work_struct *__unused)
  473. {
  474. int dir, total;
  475. mutex_lock(&hash_resize_mutex);
  476. total = 0;
  477. for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
  478. if (xfrm_bydst_should_resize(dir, &total))
  479. xfrm_bydst_resize(dir);
  480. }
  481. if (xfrm_byidx_should_resize(total))
  482. xfrm_byidx_resize(total);
  483. mutex_unlock(&hash_resize_mutex);
  484. }
  485. static DECLARE_WORK(xfrm_hash_work, xfrm_hash_resize);
  486. /* Generate new index... KAME seems to generate them ordered by cost
  487. * of an absolute inpredictability of ordering of rules. This will not pass. */
  488. static u32 xfrm_gen_index(u8 type, int dir)
  489. {
  490. static u32 idx_generator;
  491. for (;;) {
  492. struct hlist_node *entry;
  493. struct hlist_head *list;
  494. struct xfrm_policy *p;
  495. u32 idx;
  496. int found;
  497. idx = (idx_generator | dir);
  498. idx_generator += 8;
  499. if (idx == 0)
  500. idx = 8;
  501. list = xfrm_policy_byidx + idx_hash(idx);
  502. found = 0;
  503. hlist_for_each_entry(p, entry, list, byidx) {
  504. if (p->index == idx) {
  505. found = 1;
  506. break;
  507. }
  508. }
  509. if (!found)
  510. return idx;
  511. }
  512. }
  513. static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
  514. {
  515. u32 *p1 = (u32 *) s1;
  516. u32 *p2 = (u32 *) s2;
  517. int len = sizeof(struct xfrm_selector) / sizeof(u32);
  518. int i;
  519. for (i = 0; i < len; i++) {
  520. if (p1[i] != p2[i])
  521. return 1;
  522. }
  523. return 0;
  524. }
  525. int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
  526. {
  527. struct xfrm_policy *pol;
  528. struct xfrm_policy *delpol;
  529. struct hlist_head *chain;
  530. struct hlist_node *entry, *newpos;
  531. struct dst_entry *gc_list;
  532. write_lock_bh(&xfrm_policy_lock);
  533. chain = policy_hash_bysel(&policy->selector, policy->family, dir);
  534. delpol = NULL;
  535. newpos = NULL;
  536. hlist_for_each_entry(pol, entry, chain, bydst) {
  537. if (pol->type == policy->type &&
  538. !selector_cmp(&pol->selector, &policy->selector) &&
  539. xfrm_sec_ctx_match(pol->security, policy->security) &&
  540. !WARN_ON(delpol)) {
  541. if (excl) {
  542. write_unlock_bh(&xfrm_policy_lock);
  543. return -EEXIST;
  544. }
  545. delpol = pol;
  546. if (policy->priority > pol->priority)
  547. continue;
  548. } else if (policy->priority >= pol->priority) {
  549. newpos = &pol->bydst;
  550. continue;
  551. }
  552. if (delpol)
  553. break;
  554. }
  555. if (newpos)
  556. hlist_add_after(newpos, &policy->bydst);
  557. else
  558. hlist_add_head(&policy->bydst, chain);
  559. xfrm_pol_hold(policy);
  560. xfrm_policy_count[dir]++;
  561. atomic_inc(&flow_cache_genid);
  562. if (delpol) {
  563. hlist_del(&delpol->bydst);
  564. hlist_del(&delpol->byidx);
  565. xfrm_policy_count[dir]--;
  566. }
  567. policy->index = delpol ? delpol->index : xfrm_gen_index(policy->type, dir);
  568. hlist_add_head(&policy->byidx, xfrm_policy_byidx+idx_hash(policy->index));
  569. policy->curlft.add_time = (unsigned long)xtime.tv_sec;
  570. policy->curlft.use_time = 0;
  571. if (!mod_timer(&policy->timer, jiffies + HZ))
  572. xfrm_pol_hold(policy);
  573. write_unlock_bh(&xfrm_policy_lock);
  574. if (delpol)
  575. xfrm_policy_kill(delpol);
  576. else if (xfrm_bydst_should_resize(dir, NULL))
  577. schedule_work(&xfrm_hash_work);
  578. read_lock_bh(&xfrm_policy_lock);
  579. gc_list = NULL;
  580. entry = &policy->bydst;
  581. hlist_for_each_entry_continue(policy, entry, bydst) {
  582. struct dst_entry *dst;
  583. write_lock(&policy->lock);
  584. dst = policy->bundles;
  585. if (dst) {
  586. struct dst_entry *tail = dst;
  587. while (tail->next)
  588. tail = tail->next;
  589. tail->next = gc_list;
  590. gc_list = dst;
  591. policy->bundles = NULL;
  592. }
  593. write_unlock(&policy->lock);
  594. }
  595. read_unlock_bh(&xfrm_policy_lock);
  596. while (gc_list) {
  597. struct dst_entry *dst = gc_list;
  598. gc_list = dst->next;
  599. dst_free(dst);
  600. }
  601. return 0;
  602. }
  603. EXPORT_SYMBOL(xfrm_policy_insert);
  604. struct xfrm_policy *xfrm_policy_bysel_ctx(u8 type, int dir,
  605. struct xfrm_selector *sel,
  606. struct xfrm_sec_ctx *ctx, int delete,
  607. int *err)
  608. {
  609. struct xfrm_policy *pol, *ret;
  610. struct hlist_head *chain;
  611. struct hlist_node *entry;
  612. *err = 0;
  613. write_lock_bh(&xfrm_policy_lock);
  614. chain = policy_hash_bysel(sel, sel->family, dir);
  615. ret = NULL;
  616. hlist_for_each_entry(pol, entry, chain, bydst) {
  617. if (pol->type == type &&
  618. !selector_cmp(sel, &pol->selector) &&
  619. xfrm_sec_ctx_match(ctx, pol->security)) {
  620. xfrm_pol_hold(pol);
  621. if (delete) {
  622. *err = security_xfrm_policy_delete(pol);
  623. if (*err) {
  624. write_unlock_bh(&xfrm_policy_lock);
  625. return pol;
  626. }
  627. hlist_del(&pol->bydst);
  628. hlist_del(&pol->byidx);
  629. xfrm_policy_count[dir]--;
  630. }
  631. ret = pol;
  632. break;
  633. }
  634. }
  635. write_unlock_bh(&xfrm_policy_lock);
  636. if (ret && delete) {
  637. atomic_inc(&flow_cache_genid);
  638. xfrm_policy_kill(ret);
  639. }
  640. return ret;
  641. }
  642. EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
  643. struct xfrm_policy *xfrm_policy_byid(u8 type, int dir, u32 id, int delete,
  644. int *err)
  645. {
  646. struct xfrm_policy *pol, *ret;
  647. struct hlist_head *chain;
  648. struct hlist_node *entry;
  649. *err = -ENOENT;
  650. if (xfrm_policy_id2dir(id) != dir)
  651. return NULL;
  652. *err = 0;
  653. write_lock_bh(&xfrm_policy_lock);
  654. chain = xfrm_policy_byidx + idx_hash(id);
  655. ret = NULL;
  656. hlist_for_each_entry(pol, entry, chain, byidx) {
  657. if (pol->type == type && pol->index == id) {
  658. xfrm_pol_hold(pol);
  659. if (delete) {
  660. *err = security_xfrm_policy_delete(pol);
  661. if (*err) {
  662. write_unlock_bh(&xfrm_policy_lock);
  663. return pol;
  664. }
  665. hlist_del(&pol->bydst);
  666. hlist_del(&pol->byidx);
  667. xfrm_policy_count[dir]--;
  668. }
  669. ret = pol;
  670. break;
  671. }
  672. }
  673. write_unlock_bh(&xfrm_policy_lock);
  674. if (ret && delete) {
  675. atomic_inc(&flow_cache_genid);
  676. xfrm_policy_kill(ret);
  677. }
  678. return ret;
  679. }
  680. EXPORT_SYMBOL(xfrm_policy_byid);
  681. void xfrm_policy_flush(u8 type, struct xfrm_audit *audit_info)
  682. {
  683. int dir;
  684. write_lock_bh(&xfrm_policy_lock);
  685. for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
  686. struct xfrm_policy *pol;
  687. struct hlist_node *entry;
  688. int i, killed;
  689. killed = 0;
  690. again1:
  691. hlist_for_each_entry(pol, entry,
  692. &xfrm_policy_inexact[dir], bydst) {
  693. if (pol->type != type)
  694. continue;
  695. hlist_del(&pol->bydst);
  696. hlist_del(&pol->byidx);
  697. write_unlock_bh(&xfrm_policy_lock);
  698. xfrm_audit_log(audit_info->loginuid, audit_info->secid,
  699. AUDIT_MAC_IPSEC_DELSPD, 1, pol, NULL);
  700. xfrm_policy_kill(pol);
  701. killed++;
  702. write_lock_bh(&xfrm_policy_lock);
  703. goto again1;
  704. }
  705. for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
  706. again2:
  707. hlist_for_each_entry(pol, entry,
  708. xfrm_policy_bydst[dir].table + i,
  709. bydst) {
  710. if (pol->type != type)
  711. continue;
  712. hlist_del(&pol->bydst);
  713. hlist_del(&pol->byidx);
  714. write_unlock_bh(&xfrm_policy_lock);
  715. xfrm_audit_log(audit_info->loginuid,
  716. audit_info->secid,
  717. AUDIT_MAC_IPSEC_DELSPD, 1,
  718. pol, NULL);
  719. xfrm_policy_kill(pol);
  720. killed++;
  721. write_lock_bh(&xfrm_policy_lock);
  722. goto again2;
  723. }
  724. }
  725. xfrm_policy_count[dir] -= killed;
  726. }
  727. atomic_inc(&flow_cache_genid);
  728. write_unlock_bh(&xfrm_policy_lock);
  729. }
  730. EXPORT_SYMBOL(xfrm_policy_flush);
  731. int xfrm_policy_walk(u8 type, int (*func)(struct xfrm_policy *, int, int, void*),
  732. void *data)
  733. {
  734. struct xfrm_policy *pol, *last = NULL;
  735. struct hlist_node *entry;
  736. int dir, last_dir = 0, count, error;
  737. read_lock_bh(&xfrm_policy_lock);
  738. count = 0;
  739. for (dir = 0; dir < 2*XFRM_POLICY_MAX; dir++) {
  740. struct hlist_head *table = xfrm_policy_bydst[dir].table;
  741. int i;
  742. hlist_for_each_entry(pol, entry,
  743. &xfrm_policy_inexact[dir], bydst) {
  744. if (pol->type != type)
  745. continue;
  746. if (last) {
  747. error = func(last, last_dir % XFRM_POLICY_MAX,
  748. count, data);
  749. if (error)
  750. goto out;
  751. }
  752. last = pol;
  753. last_dir = dir;
  754. count++;
  755. }
  756. for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
  757. hlist_for_each_entry(pol, entry, table + i, bydst) {
  758. if (pol->type != type)
  759. continue;
  760. if (last) {
  761. error = func(last, last_dir % XFRM_POLICY_MAX,
  762. count, data);
  763. if (error)
  764. goto out;
  765. }
  766. last = pol;
  767. last_dir = dir;
  768. count++;
  769. }
  770. }
  771. }
  772. if (count == 0) {
  773. error = -ENOENT;
  774. goto out;
  775. }
  776. error = func(last, last_dir % XFRM_POLICY_MAX, 0, data);
  777. out:
  778. read_unlock_bh(&xfrm_policy_lock);
  779. return error;
  780. }
  781. EXPORT_SYMBOL(xfrm_policy_walk);
  782. /*
  783. * Find policy to apply to this flow.
  784. *
  785. * Returns 0 if policy found, else an -errno.
  786. */
  787. static int xfrm_policy_match(struct xfrm_policy *pol, struct flowi *fl,
  788. u8 type, u16 family, int dir)
  789. {
  790. struct xfrm_selector *sel = &pol->selector;
  791. int match, ret = -ESRCH;
  792. if (pol->family != family ||
  793. pol->type != type)
  794. return ret;
  795. match = xfrm_selector_match(sel, fl, family);
  796. if (match)
  797. ret = security_xfrm_policy_lookup(pol, fl->secid, dir);
  798. return ret;
  799. }
  800. static struct xfrm_policy *xfrm_policy_lookup_bytype(u8 type, struct flowi *fl,
  801. u16 family, u8 dir)
  802. {
  803. int err;
  804. struct xfrm_policy *pol, *ret;
  805. xfrm_address_t *daddr, *saddr;
  806. struct hlist_node *entry;
  807. struct hlist_head *chain;
  808. u32 priority = ~0U;
  809. daddr = xfrm_flowi_daddr(fl, family);
  810. saddr = xfrm_flowi_saddr(fl, family);
  811. if (unlikely(!daddr || !saddr))
  812. return NULL;
  813. read_lock_bh(&xfrm_policy_lock);
  814. chain = policy_hash_direct(daddr, saddr, family, dir);
  815. ret = NULL;
  816. hlist_for_each_entry(pol, entry, chain, bydst) {
  817. err = xfrm_policy_match(pol, fl, type, family, dir);
  818. if (err) {
  819. if (err == -ESRCH)
  820. continue;
  821. else {
  822. ret = ERR_PTR(err);
  823. goto fail;
  824. }
  825. } else {
  826. ret = pol;
  827. priority = ret->priority;
  828. break;
  829. }
  830. }
  831. chain = &xfrm_policy_inexact[dir];
  832. hlist_for_each_entry(pol, entry, chain, bydst) {
  833. err = xfrm_policy_match(pol, fl, type, family, dir);
  834. if (err) {
  835. if (err == -ESRCH)
  836. continue;
  837. else {
  838. ret = ERR_PTR(err);
  839. goto fail;
  840. }
  841. } else if (pol->priority < priority) {
  842. ret = pol;
  843. break;
  844. }
  845. }
  846. if (ret)
  847. xfrm_pol_hold(ret);
  848. fail:
  849. read_unlock_bh(&xfrm_policy_lock);
  850. return ret;
  851. }
  852. static int xfrm_policy_lookup(struct flowi *fl, u16 family, u8 dir,
  853. void **objp, atomic_t **obj_refp)
  854. {
  855. struct xfrm_policy *pol;
  856. int err = 0;
  857. #ifdef CONFIG_XFRM_SUB_POLICY
  858. pol = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_SUB, fl, family, dir);
  859. if (IS_ERR(pol)) {
  860. err = PTR_ERR(pol);
  861. pol = NULL;
  862. }
  863. if (pol || err)
  864. goto end;
  865. #endif
  866. pol = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_MAIN, fl, family, dir);
  867. if (IS_ERR(pol)) {
  868. err = PTR_ERR(pol);
  869. pol = NULL;
  870. }
  871. #ifdef CONFIG_XFRM_SUB_POLICY
  872. end:
  873. #endif
  874. if ((*objp = (void *) pol) != NULL)
  875. *obj_refp = &pol->refcnt;
  876. return err;
  877. }
  878. static inline int policy_to_flow_dir(int dir)
  879. {
  880. if (XFRM_POLICY_IN == FLOW_DIR_IN &&
  881. XFRM_POLICY_OUT == FLOW_DIR_OUT &&
  882. XFRM_POLICY_FWD == FLOW_DIR_FWD)
  883. return dir;
  884. switch (dir) {
  885. default:
  886. case XFRM_POLICY_IN:
  887. return FLOW_DIR_IN;
  888. case XFRM_POLICY_OUT:
  889. return FLOW_DIR_OUT;
  890. case XFRM_POLICY_FWD:
  891. return FLOW_DIR_FWD;
  892. };
  893. }
  894. static struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir, struct flowi *fl)
  895. {
  896. struct xfrm_policy *pol;
  897. read_lock_bh(&xfrm_policy_lock);
  898. if ((pol = sk->sk_policy[dir]) != NULL) {
  899. int match = xfrm_selector_match(&pol->selector, fl,
  900. sk->sk_family);
  901. int err = 0;
  902. if (match) {
  903. err = security_xfrm_policy_lookup(pol, fl->secid,
  904. policy_to_flow_dir(dir));
  905. if (!err)
  906. xfrm_pol_hold(pol);
  907. else if (err == -ESRCH)
  908. pol = NULL;
  909. else
  910. pol = ERR_PTR(err);
  911. } else
  912. pol = NULL;
  913. }
  914. read_unlock_bh(&xfrm_policy_lock);
  915. return pol;
  916. }
  917. static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
  918. {
  919. struct hlist_head *chain = policy_hash_bysel(&pol->selector,
  920. pol->family, dir);
  921. hlist_add_head(&pol->bydst, chain);
  922. hlist_add_head(&pol->byidx, xfrm_policy_byidx+idx_hash(pol->index));
  923. xfrm_policy_count[dir]++;
  924. xfrm_pol_hold(pol);
  925. if (xfrm_bydst_should_resize(dir, NULL))
  926. schedule_work(&xfrm_hash_work);
  927. }
  928. static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
  929. int dir)
  930. {
  931. if (hlist_unhashed(&pol->bydst))
  932. return NULL;
  933. hlist_del(&pol->bydst);
  934. hlist_del(&pol->byidx);
  935. xfrm_policy_count[dir]--;
  936. return pol;
  937. }
  938. int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
  939. {
  940. write_lock_bh(&xfrm_policy_lock);
  941. pol = __xfrm_policy_unlink(pol, dir);
  942. write_unlock_bh(&xfrm_policy_lock);
  943. if (pol) {
  944. if (dir < XFRM_POLICY_MAX)
  945. atomic_inc(&flow_cache_genid);
  946. xfrm_policy_kill(pol);
  947. return 0;
  948. }
  949. return -ENOENT;
  950. }
  951. EXPORT_SYMBOL(xfrm_policy_delete);
  952. int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
  953. {
  954. struct xfrm_policy *old_pol;
  955. #ifdef CONFIG_XFRM_SUB_POLICY
  956. if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
  957. return -EINVAL;
  958. #endif
  959. write_lock_bh(&xfrm_policy_lock);
  960. old_pol = sk->sk_policy[dir];
  961. sk->sk_policy[dir] = pol;
  962. if (pol) {
  963. pol->curlft.add_time = (unsigned long)xtime.tv_sec;
  964. pol->index = xfrm_gen_index(pol->type, XFRM_POLICY_MAX+dir);
  965. __xfrm_policy_link(pol, XFRM_POLICY_MAX+dir);
  966. }
  967. if (old_pol)
  968. __xfrm_policy_unlink(old_pol, XFRM_POLICY_MAX+dir);
  969. write_unlock_bh(&xfrm_policy_lock);
  970. if (old_pol) {
  971. xfrm_policy_kill(old_pol);
  972. }
  973. return 0;
  974. }
  975. static struct xfrm_policy *clone_policy(struct xfrm_policy *old, int dir)
  976. {
  977. struct xfrm_policy *newp = xfrm_policy_alloc(GFP_ATOMIC);
  978. if (newp) {
  979. newp->selector = old->selector;
  980. if (security_xfrm_policy_clone(old, newp)) {
  981. kfree(newp);
  982. return NULL; /* ENOMEM */
  983. }
  984. newp->lft = old->lft;
  985. newp->curlft = old->curlft;
  986. newp->action = old->action;
  987. newp->flags = old->flags;
  988. newp->xfrm_nr = old->xfrm_nr;
  989. newp->index = old->index;
  990. newp->type = old->type;
  991. memcpy(newp->xfrm_vec, old->xfrm_vec,
  992. newp->xfrm_nr*sizeof(struct xfrm_tmpl));
  993. write_lock_bh(&xfrm_policy_lock);
  994. __xfrm_policy_link(newp, XFRM_POLICY_MAX+dir);
  995. write_unlock_bh(&xfrm_policy_lock);
  996. xfrm_pol_put(newp);
  997. }
  998. return newp;
  999. }
  1000. int __xfrm_sk_clone_policy(struct sock *sk)
  1001. {
  1002. struct xfrm_policy *p0 = sk->sk_policy[0],
  1003. *p1 = sk->sk_policy[1];
  1004. sk->sk_policy[0] = sk->sk_policy[1] = NULL;
  1005. if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL)
  1006. return -ENOMEM;
  1007. if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL)
  1008. return -ENOMEM;
  1009. return 0;
  1010. }
  1011. static int
  1012. xfrm_get_saddr(xfrm_address_t *local, xfrm_address_t *remote,
  1013. unsigned short family)
  1014. {
  1015. int err;
  1016. struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
  1017. if (unlikely(afinfo == NULL))
  1018. return -EINVAL;
  1019. err = afinfo->get_saddr(local, remote);
  1020. xfrm_policy_put_afinfo(afinfo);
  1021. return err;
  1022. }
  1023. /* Resolve list of templates for the flow, given policy. */
  1024. static int
  1025. xfrm_tmpl_resolve_one(struct xfrm_policy *policy, struct flowi *fl,
  1026. struct xfrm_state **xfrm,
  1027. unsigned short family)
  1028. {
  1029. int nx;
  1030. int i, error;
  1031. xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
  1032. xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
  1033. xfrm_address_t tmp;
  1034. for (nx=0, i = 0; i < policy->xfrm_nr; i++) {
  1035. struct xfrm_state *x;
  1036. xfrm_address_t *remote = daddr;
  1037. xfrm_address_t *local = saddr;
  1038. struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
  1039. if (tmpl->mode == XFRM_MODE_TUNNEL) {
  1040. remote = &tmpl->id.daddr;
  1041. local = &tmpl->saddr;
  1042. family = tmpl->encap_family;
  1043. if (xfrm_addr_any(local, family)) {
  1044. error = xfrm_get_saddr(&tmp, remote, family);
  1045. if (error)
  1046. goto fail;
  1047. local = &tmp;
  1048. }
  1049. }
  1050. x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);
  1051. if (x && x->km.state == XFRM_STATE_VALID) {
  1052. xfrm[nx++] = x;
  1053. daddr = remote;
  1054. saddr = local;
  1055. continue;
  1056. }
  1057. if (x) {
  1058. error = (x->km.state == XFRM_STATE_ERROR ?
  1059. -EINVAL : -EAGAIN);
  1060. xfrm_state_put(x);
  1061. }
  1062. if (!tmpl->optional)
  1063. goto fail;
  1064. }
  1065. return nx;
  1066. fail:
  1067. for (nx--; nx>=0; nx--)
  1068. xfrm_state_put(xfrm[nx]);
  1069. return error;
  1070. }
  1071. static int
  1072. xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, struct flowi *fl,
  1073. struct xfrm_state **xfrm,
  1074. unsigned short family)
  1075. {
  1076. struct xfrm_state *tp[XFRM_MAX_DEPTH];
  1077. struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
  1078. int cnx = 0;
  1079. int error;
  1080. int ret;
  1081. int i;
  1082. for (i = 0; i < npols; i++) {
  1083. if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
  1084. error = -ENOBUFS;
  1085. goto fail;
  1086. }
  1087. ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
  1088. if (ret < 0) {
  1089. error = ret;
  1090. goto fail;
  1091. } else
  1092. cnx += ret;
  1093. }
  1094. /* found states are sorted for outbound processing */
  1095. if (npols > 1)
  1096. xfrm_state_sort(xfrm, tpp, cnx, family);
  1097. return cnx;
  1098. fail:
  1099. for (cnx--; cnx>=0; cnx--)
  1100. xfrm_state_put(tpp[cnx]);
  1101. return error;
  1102. }
  1103. /* Check that the bundle accepts the flow and its components are
  1104. * still valid.
  1105. */
  1106. static struct dst_entry *
  1107. xfrm_find_bundle(struct flowi *fl, struct xfrm_policy *policy, unsigned short family)
  1108. {
  1109. struct dst_entry *x;
  1110. struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
  1111. if (unlikely(afinfo == NULL))
  1112. return ERR_PTR(-EINVAL);
  1113. x = afinfo->find_bundle(fl, policy);
  1114. xfrm_policy_put_afinfo(afinfo);
  1115. return x;
  1116. }
  1117. /* Allocate chain of dst_entry's, attach known xfrm's, calculate
  1118. * all the metrics... Shortly, bundle a bundle.
  1119. */
  1120. static int
  1121. xfrm_bundle_create(struct xfrm_policy *policy, struct xfrm_state **xfrm, int nx,
  1122. struct flowi *fl, struct dst_entry **dst_p,
  1123. unsigned short family)
  1124. {
  1125. int err;
  1126. struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
  1127. if (unlikely(afinfo == NULL))
  1128. return -EINVAL;
  1129. err = afinfo->bundle_create(policy, xfrm, nx, fl, dst_p);
  1130. xfrm_policy_put_afinfo(afinfo);
  1131. return err;
  1132. }
  1133. static int stale_bundle(struct dst_entry *dst);
  1134. /* Main function: finds/creates a bundle for given flow.
  1135. *
  1136. * At the moment we eat a raw IP route. Mostly to speed up lookups
  1137. * on interfaces with disabled IPsec.
  1138. */
  1139. int xfrm_lookup(struct dst_entry **dst_p, struct flowi *fl,
  1140. struct sock *sk, int flags)
  1141. {
  1142. struct xfrm_policy *policy;
  1143. struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
  1144. int npols;
  1145. int pol_dead;
  1146. int xfrm_nr;
  1147. int pi;
  1148. struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
  1149. struct dst_entry *dst, *dst_orig = *dst_p;
  1150. int nx = 0;
  1151. int err;
  1152. u32 genid;
  1153. u16 family;
  1154. u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
  1155. restart:
  1156. genid = atomic_read(&flow_cache_genid);
  1157. policy = NULL;
  1158. for (pi = 0; pi < ARRAY_SIZE(pols); pi++)
  1159. pols[pi] = NULL;
  1160. npols = 0;
  1161. pol_dead = 0;
  1162. xfrm_nr = 0;
  1163. if (sk && sk->sk_policy[1]) {
  1164. policy = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl);
  1165. if (IS_ERR(policy))
  1166. return PTR_ERR(policy);
  1167. }
  1168. if (!policy) {
  1169. /* To accelerate a bit... */
  1170. if ((dst_orig->flags & DST_NOXFRM) ||
  1171. !xfrm_policy_count[XFRM_POLICY_OUT])
  1172. return 0;
  1173. policy = flow_cache_lookup(fl, dst_orig->ops->family,
  1174. dir, xfrm_policy_lookup);
  1175. if (IS_ERR(policy))
  1176. return PTR_ERR(policy);
  1177. }
  1178. if (!policy)
  1179. return 0;
  1180. family = dst_orig->ops->family;
  1181. policy->curlft.use_time = (unsigned long)xtime.tv_sec;
  1182. pols[0] = policy;
  1183. npols ++;
  1184. xfrm_nr += pols[0]->xfrm_nr;
  1185. switch (policy->action) {
  1186. case XFRM_POLICY_BLOCK:
  1187. /* Prohibit the flow */
  1188. err = -EPERM;
  1189. goto error;
  1190. case XFRM_POLICY_ALLOW:
  1191. #ifndef CONFIG_XFRM_SUB_POLICY
  1192. if (policy->xfrm_nr == 0) {
  1193. /* Flow passes not transformed. */
  1194. xfrm_pol_put(policy);
  1195. return 0;
  1196. }
  1197. #endif
  1198. /* Try to find matching bundle.
  1199. *
  1200. * LATER: help from flow cache. It is optional, this
  1201. * is required only for output policy.
  1202. */
  1203. dst = xfrm_find_bundle(fl, policy, family);
  1204. if (IS_ERR(dst)) {
  1205. err = PTR_ERR(dst);
  1206. goto error;
  1207. }
  1208. if (dst)
  1209. break;
  1210. #ifdef CONFIG_XFRM_SUB_POLICY
  1211. if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
  1212. pols[1] = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_MAIN,
  1213. fl, family,
  1214. XFRM_POLICY_OUT);
  1215. if (pols[1]) {
  1216. if (IS_ERR(pols[1])) {
  1217. err = PTR_ERR(pols[1]);
  1218. goto error;
  1219. }
  1220. if (pols[1]->action == XFRM_POLICY_BLOCK) {
  1221. err = -EPERM;
  1222. goto error;
  1223. }
  1224. npols ++;
  1225. xfrm_nr += pols[1]->xfrm_nr;
  1226. }
  1227. }
  1228. /*
  1229. * Because neither flowi nor bundle information knows about
  1230. * transformation template size. On more than one policy usage
  1231. * we can realize whether all of them is bypass or not after
  1232. * they are searched. See above not-transformed bypass
  1233. * is surrounded by non-sub policy configuration, too.
  1234. */
  1235. if (xfrm_nr == 0) {
  1236. /* Flow passes not transformed. */
  1237. xfrm_pols_put(pols, npols);
  1238. return 0;
  1239. }
  1240. #endif
  1241. nx = xfrm_tmpl_resolve(pols, npols, fl, xfrm, family);
  1242. if (unlikely(nx<0)) {
  1243. err = nx;
  1244. if (err == -EAGAIN && flags) {
  1245. DECLARE_WAITQUEUE(wait, current);
  1246. add_wait_queue(&km_waitq, &wait);
  1247. set_current_state(TASK_INTERRUPTIBLE);
  1248. schedule();
  1249. set_current_state(TASK_RUNNING);
  1250. remove_wait_queue(&km_waitq, &wait);
  1251. nx = xfrm_tmpl_resolve(pols, npols, fl, xfrm, family);
  1252. if (nx == -EAGAIN && signal_pending(current)) {
  1253. err = -ERESTART;
  1254. goto error;
  1255. }
  1256. if (nx == -EAGAIN ||
  1257. genid != atomic_read(&flow_cache_genid)) {
  1258. xfrm_pols_put(pols, npols);
  1259. goto restart;
  1260. }
  1261. err = nx;
  1262. }
  1263. if (err < 0)
  1264. goto error;
  1265. }
  1266. if (nx == 0) {
  1267. /* Flow passes not transformed. */
  1268. xfrm_pols_put(pols, npols);
  1269. return 0;
  1270. }
  1271. dst = dst_orig;
  1272. err = xfrm_bundle_create(policy, xfrm, nx, fl, &dst, family);
  1273. if (unlikely(err)) {
  1274. int i;
  1275. for (i=0; i<nx; i++)
  1276. xfrm_state_put(xfrm[i]);
  1277. goto error;
  1278. }
  1279. for (pi = 0; pi < npols; pi++) {
  1280. read_lock_bh(&pols[pi]->lock);
  1281. pol_dead |= pols[pi]->dead;
  1282. read_unlock_bh(&pols[pi]->lock);
  1283. }
  1284. write_lock_bh(&policy->lock);
  1285. if (unlikely(pol_dead || stale_bundle(dst))) {
  1286. /* Wow! While we worked on resolving, this
  1287. * policy has gone. Retry. It is not paranoia,
  1288. * we just cannot enlist new bundle to dead object.
  1289. * We can't enlist stable bundles either.
  1290. */
  1291. write_unlock_bh(&policy->lock);
  1292. if (dst)
  1293. dst_free(dst);
  1294. err = -EHOSTUNREACH;
  1295. goto error;
  1296. }
  1297. dst->next = policy->bundles;
  1298. policy->bundles = dst;
  1299. dst_hold(dst);
  1300. write_unlock_bh(&policy->lock);
  1301. }
  1302. *dst_p = dst;
  1303. dst_release(dst_orig);
  1304. xfrm_pols_put(pols, npols);
  1305. return 0;
  1306. error:
  1307. dst_release(dst_orig);
  1308. xfrm_pols_put(pols, npols);
  1309. *dst_p = NULL;
  1310. return err;
  1311. }
  1312. EXPORT_SYMBOL(xfrm_lookup);
  1313. static inline int
  1314. xfrm_secpath_reject(int idx, struct sk_buff *skb, struct flowi *fl)
  1315. {
  1316. struct xfrm_state *x;
  1317. int err;
  1318. if (!skb->sp || idx < 0 || idx >= skb->sp->len)
  1319. return 0;
  1320. x = skb->sp->xvec[idx];
  1321. if (!x->type->reject)
  1322. return 0;
  1323. xfrm_state_hold(x);
  1324. err = x->type->reject(x, skb, fl);
  1325. xfrm_state_put(x);
  1326. return err;
  1327. }
  1328. /* When skb is transformed back to its "native" form, we have to
  1329. * check policy restrictions. At the moment we make this in maximally
  1330. * stupid way. Shame on me. :-) Of course, connected sockets must
  1331. * have policy cached at them.
  1332. */
  1333. static inline int
  1334. xfrm_state_ok(struct xfrm_tmpl *tmpl, struct xfrm_state *x,
  1335. unsigned short family)
  1336. {
  1337. if (xfrm_state_kern(x))
  1338. return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
  1339. return x->id.proto == tmpl->id.proto &&
  1340. (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
  1341. (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
  1342. x->props.mode == tmpl->mode &&
  1343. ((tmpl->aalgos & (1<<x->props.aalgo)) ||
  1344. !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
  1345. !(x->props.mode != XFRM_MODE_TRANSPORT &&
  1346. xfrm_state_addr_cmp(tmpl, x, family));
  1347. }
  1348. /*
  1349. * 0 or more than 0 is returned when validation is succeeded (either bypass
  1350. * because of optional transport mode, or next index of the mathced secpath
  1351. * state with the template.
  1352. * -1 is returned when no matching template is found.
  1353. * Otherwise "-2 - errored_index" is returned.
  1354. */
  1355. static inline int
  1356. xfrm_policy_ok(struct xfrm_tmpl *tmpl, struct sec_path *sp, int start,
  1357. unsigned short family)
  1358. {
  1359. int idx = start;
  1360. if (tmpl->optional) {
  1361. if (tmpl->mode == XFRM_MODE_TRANSPORT)
  1362. return start;
  1363. } else
  1364. start = -1;
  1365. for (; idx < sp->len; idx++) {
  1366. if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
  1367. return ++idx;
  1368. if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
  1369. if (start == -1)
  1370. start = -2-idx;
  1371. break;
  1372. }
  1373. }
  1374. return start;
  1375. }
  1376. int
  1377. xfrm_decode_session(struct sk_buff *skb, struct flowi *fl, unsigned short family)
  1378. {
  1379. struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
  1380. int err;
  1381. if (unlikely(afinfo == NULL))
  1382. return -EAFNOSUPPORT;
  1383. afinfo->decode_session(skb, fl);
  1384. err = security_xfrm_decode_session(skb, &fl->secid);
  1385. xfrm_policy_put_afinfo(afinfo);
  1386. return err;
  1387. }
  1388. EXPORT_SYMBOL(xfrm_decode_session);
  1389. static inline int secpath_has_nontransport(struct sec_path *sp, int k, int *idxp)
  1390. {
  1391. for (; k < sp->len; k++) {
  1392. if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
  1393. *idxp = k;
  1394. return 1;
  1395. }
  1396. }
  1397. return 0;
  1398. }
  1399. int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
  1400. unsigned short family)
  1401. {
  1402. struct xfrm_policy *pol;
  1403. struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
  1404. int npols = 0;
  1405. int xfrm_nr;
  1406. int pi;
  1407. struct flowi fl;
  1408. u8 fl_dir = policy_to_flow_dir(dir);
  1409. int xerr_idx = -1;
  1410. if (xfrm_decode_session(skb, &fl, family) < 0)
  1411. return 0;
  1412. nf_nat_decode_session(skb, &fl, family);
  1413. /* First, check used SA against their selectors. */
  1414. if (skb->sp) {
  1415. int i;
  1416. for (i=skb->sp->len-1; i>=0; i--) {
  1417. struct xfrm_state *x = skb->sp->xvec[i];
  1418. if (!xfrm_selector_match(&x->sel, &fl, family))
  1419. return 0;
  1420. }
  1421. }
  1422. pol = NULL;
  1423. if (sk && sk->sk_policy[dir]) {
  1424. pol = xfrm_sk_policy_lookup(sk, dir, &fl);
  1425. if (IS_ERR(pol))
  1426. return 0;
  1427. }
  1428. if (!pol)
  1429. pol = flow_cache_lookup(&fl, family, fl_dir,
  1430. xfrm_policy_lookup);
  1431. if (IS_ERR(pol))
  1432. return 0;
  1433. if (!pol) {
  1434. if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
  1435. xfrm_secpath_reject(xerr_idx, skb, &fl);
  1436. return 0;
  1437. }
  1438. return 1;
  1439. }
  1440. pol->curlft.use_time = (unsigned long)xtime.tv_sec;
  1441. pols[0] = pol;
  1442. npols ++;
  1443. #ifdef CONFIG_XFRM_SUB_POLICY
  1444. if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
  1445. pols[1] = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_MAIN,
  1446. &fl, family,
  1447. XFRM_POLICY_IN);
  1448. if (pols[1]) {
  1449. if (IS_ERR(pols[1]))
  1450. return 0;
  1451. pols[1]->curlft.use_time = (unsigned long)xtime.tv_sec;
  1452. npols ++;
  1453. }
  1454. }
  1455. #endif
  1456. if (pol->action == XFRM_POLICY_ALLOW) {
  1457. struct sec_path *sp;
  1458. static struct sec_path dummy;
  1459. struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
  1460. struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
  1461. struct xfrm_tmpl **tpp = tp;
  1462. int ti = 0;
  1463. int i, k;
  1464. if ((sp = skb->sp) == NULL)
  1465. sp = &dummy;
  1466. for (pi = 0; pi < npols; pi++) {
  1467. if (pols[pi] != pol &&
  1468. pols[pi]->action != XFRM_POLICY_ALLOW)
  1469. goto reject;
  1470. if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH)
  1471. goto reject_error;
  1472. for (i = 0; i < pols[pi]->xfrm_nr; i++)
  1473. tpp[ti++] = &pols[pi]->xfrm_vec[i];
  1474. }
  1475. xfrm_nr = ti;
  1476. if (npols > 1) {
  1477. xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
  1478. tpp = stp;
  1479. }
  1480. /* For each tunnel xfrm, find the first matching tmpl.
  1481. * For each tmpl before that, find corresponding xfrm.
  1482. * Order is _important_. Later we will implement
  1483. * some barriers, but at the moment barriers
  1484. * are implied between each two transformations.
  1485. */
  1486. for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
  1487. k = xfrm_policy_ok(tpp[i], sp, k, family);
  1488. if (k < 0) {
  1489. if (k < -1)
  1490. /* "-2 - errored_index" returned */
  1491. xerr_idx = -(2+k);
  1492. goto reject;
  1493. }
  1494. }
  1495. if (secpath_has_nontransport(sp, k, &xerr_idx))
  1496. goto reject;
  1497. xfrm_pols_put(pols, npols);
  1498. return 1;
  1499. }
  1500. reject:
  1501. xfrm_secpath_reject(xerr_idx, skb, &fl);
  1502. reject_error:
  1503. xfrm_pols_put(pols, npols);
  1504. return 0;
  1505. }
  1506. EXPORT_SYMBOL(__xfrm_policy_check);
  1507. int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
  1508. {
  1509. struct flowi fl;
  1510. if (xfrm_decode_session(skb, &fl, family) < 0)
  1511. return 0;
  1512. return xfrm_lookup(&skb->dst, &fl, NULL, 0) == 0;
  1513. }
  1514. EXPORT_SYMBOL(__xfrm_route_forward);
  1515. /* Optimize later using cookies and generation ids. */
  1516. static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
  1517. {
  1518. /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
  1519. * to "-1" to force all XFRM destinations to get validated by
  1520. * dst_ops->check on every use. We do this because when a
  1521. * normal route referenced by an XFRM dst is obsoleted we do
  1522. * not go looking around for all parent referencing XFRM dsts
  1523. * so that we can invalidate them. It is just too much work.
  1524. * Instead we make the checks here on every use. For example:
  1525. *
  1526. * XFRM dst A --> IPv4 dst X
  1527. *
  1528. * X is the "xdst->route" of A (X is also the "dst->path" of A
  1529. * in this example). If X is marked obsolete, "A" will not
  1530. * notice. That's what we are validating here via the
  1531. * stale_bundle() check.
  1532. *
  1533. * When a policy's bundle is pruned, we dst_free() the XFRM
  1534. * dst which causes it's ->obsolete field to be set to a
  1535. * positive non-zero integer. If an XFRM dst has been pruned
  1536. * like this, we want to force a new route lookup.
  1537. */
  1538. if (dst->obsolete < 0 && !stale_bundle(dst))
  1539. return dst;
  1540. return NULL;
  1541. }
  1542. static int stale_bundle(struct dst_entry *dst)
  1543. {
  1544. return !xfrm_bundle_ok(NULL, (struct xfrm_dst *)dst, NULL, AF_UNSPEC, 0);
  1545. }
  1546. void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
  1547. {
  1548. while ((dst = dst->child) && dst->xfrm && dst->dev == dev) {
  1549. dst->dev = &loopback_dev;
  1550. dev_hold(&loopback_dev);
  1551. dev_put(dev);
  1552. }
  1553. }
  1554. EXPORT_SYMBOL(xfrm_dst_ifdown);
  1555. static void xfrm_link_failure(struct sk_buff *skb)
  1556. {
  1557. /* Impossible. Such dst must be popped before reaches point of failure. */
  1558. return;
  1559. }
  1560. static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
  1561. {
  1562. if (dst) {
  1563. if (dst->obsolete) {
  1564. dst_release(dst);
  1565. dst = NULL;
  1566. }
  1567. }
  1568. return dst;
  1569. }
  1570. static void prune_one_bundle(struct xfrm_policy *pol, int (*func)(struct dst_entry *), struct dst_entry **gc_list_p)
  1571. {
  1572. struct dst_entry *dst, **dstp;
  1573. write_lock(&pol->lock);
  1574. dstp = &pol->bundles;
  1575. while ((dst=*dstp) != NULL) {
  1576. if (func(dst)) {
  1577. *dstp = dst->next;
  1578. dst->next = *gc_list_p;
  1579. *gc_list_p = dst;
  1580. } else {
  1581. dstp = &dst->next;
  1582. }
  1583. }
  1584. write_unlock(&pol->lock);
  1585. }
  1586. static void xfrm_prune_bundles(int (*func)(struct dst_entry *))
  1587. {
  1588. struct dst_entry *gc_list = NULL;
  1589. int dir;
  1590. read_lock_bh(&xfrm_policy_lock);
  1591. for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
  1592. struct xfrm_policy *pol;
  1593. struct hlist_node *entry;
  1594. struct hlist_head *table;
  1595. int i;
  1596. hlist_for_each_entry(pol, entry,
  1597. &xfrm_policy_inexact[dir], bydst)
  1598. prune_one_bundle(pol, func, &gc_list);
  1599. table = xfrm_policy_bydst[dir].table;
  1600. for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
  1601. hlist_for_each_entry(pol, entry, table + i, bydst)
  1602. prune_one_bundle(pol, func, &gc_list);
  1603. }
  1604. }
  1605. read_unlock_bh(&xfrm_policy_lock);
  1606. while (gc_list) {
  1607. struct dst_entry *dst = gc_list;
  1608. gc_list = dst->next;
  1609. dst_free(dst);
  1610. }
  1611. }
  1612. static int unused_bundle(struct dst_entry *dst)
  1613. {
  1614. return !atomic_read(&dst->__refcnt);
  1615. }
  1616. static void __xfrm_garbage_collect(void)
  1617. {
  1618. xfrm_prune_bundles(unused_bundle);
  1619. }
  1620. static int xfrm_flush_bundles(void)
  1621. {
  1622. xfrm_prune_bundles(stale_bundle);
  1623. return 0;
  1624. }
  1625. void xfrm_init_pmtu(struct dst_entry *dst)
  1626. {
  1627. do {
  1628. struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
  1629. u32 pmtu, route_mtu_cached;
  1630. pmtu = dst_mtu(dst->child);
  1631. xdst->child_mtu_cached = pmtu;
  1632. pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
  1633. route_mtu_cached = dst_mtu(xdst->route);
  1634. xdst->route_mtu_cached = route_mtu_cached;
  1635. if (pmtu > route_mtu_cached)
  1636. pmtu = route_mtu_cached;
  1637. dst->metrics[RTAX_MTU-1] = pmtu;
  1638. } while ((dst = dst->next));
  1639. }
  1640. EXPORT_SYMBOL(xfrm_init_pmtu);
  1641. /* Check that the bundle accepts the flow and its components are
  1642. * still valid.
  1643. */
  1644. int xfrm_bundle_ok(struct xfrm_policy *pol, struct xfrm_dst *first,
  1645. struct flowi *fl, int family, int strict)
  1646. {
  1647. struct dst_entry *dst = &first->u.dst;
  1648. struct xfrm_dst *last;
  1649. u32 mtu;
  1650. if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) ||
  1651. (dst->dev && !netif_running(dst->dev)))
  1652. return 0;
  1653. last = NULL;
  1654. do {
  1655. struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
  1656. if (fl && !xfrm_selector_match(&dst->xfrm->sel, fl, family))
  1657. return 0;
  1658. if (fl && pol &&
  1659. !security_xfrm_state_pol_flow_match(dst->xfrm, pol, fl))
  1660. return 0;
  1661. if (dst->xfrm->km.state != XFRM_STATE_VALID)
  1662. return 0;
  1663. if (xdst->genid != dst->xfrm->genid)
  1664. return 0;
  1665. if (strict && fl && dst->xfrm->props.mode != XFRM_MODE_TUNNEL &&
  1666. !xfrm_state_addr_flow_check(dst->xfrm, fl, family))
  1667. return 0;
  1668. mtu = dst_mtu(dst->child);
  1669. if (xdst->child_mtu_cached != mtu) {
  1670. last = xdst;
  1671. xdst->child_mtu_cached = mtu;
  1672. }
  1673. if (!dst_check(xdst->route, xdst->route_cookie))
  1674. return 0;
  1675. mtu = dst_mtu(xdst->route);
  1676. if (xdst->route_mtu_cached != mtu) {
  1677. last = xdst;
  1678. xdst->route_mtu_cached = mtu;
  1679. }
  1680. dst = dst->child;
  1681. } while (dst->xfrm);
  1682. if (likely(!last))
  1683. return 1;
  1684. mtu = last->child_mtu_cached;
  1685. for (;;) {
  1686. dst = &last->u.dst;
  1687. mtu = xfrm_state_mtu(dst->xfrm, mtu);
  1688. if (mtu > last->route_mtu_cached)
  1689. mtu = last->route_mtu_cached;
  1690. dst->metrics[RTAX_MTU-1] = mtu;
  1691. if (last == first)
  1692. break;
  1693. last = last->u.next;
  1694. last->child_mtu_cached = mtu;
  1695. }
  1696. return 1;
  1697. }
  1698. EXPORT_SYMBOL(xfrm_bundle_ok);
  1699. #ifdef CONFIG_AUDITSYSCALL
  1700. /* Audit addition and deletion of SAs and ipsec policy */
  1701. void xfrm_audit_log(uid_t auid, u32 sid, int type, int result,
  1702. struct xfrm_policy *xp, struct xfrm_state *x)
  1703. {
  1704. char *secctx;
  1705. u32 secctx_len;
  1706. struct xfrm_sec_ctx *sctx = NULL;
  1707. struct audit_buffer *audit_buf;
  1708. int family;
  1709. extern int audit_enabled;
  1710. if (audit_enabled == 0)
  1711. return;
  1712. BUG_ON((type == AUDIT_MAC_IPSEC_ADDSA ||
  1713. type == AUDIT_MAC_IPSEC_DELSA) && !x);
  1714. BUG_ON((type == AUDIT_MAC_IPSEC_ADDSPD ||
  1715. type == AUDIT_MAC_IPSEC_DELSPD) && !xp);
  1716. audit_buf = audit_log_start(current->audit_context, GFP_ATOMIC, type);
  1717. if (audit_buf == NULL)
  1718. return;
  1719. switch(type) {
  1720. case AUDIT_MAC_IPSEC_ADDSA:
  1721. audit_log_format(audit_buf, "SAD add: auid=%u", auid);
  1722. break;
  1723. case AUDIT_MAC_IPSEC_DELSA:
  1724. audit_log_format(audit_buf, "SAD delete: auid=%u", auid);
  1725. break;
  1726. case AUDIT_MAC_IPSEC_ADDSPD:
  1727. audit_log_format(audit_buf, "SPD add: auid=%u", auid);
  1728. break;
  1729. case AUDIT_MAC_IPSEC_DELSPD:
  1730. audit_log_format(audit_buf, "SPD delete: auid=%u", auid);
  1731. break;
  1732. default:
  1733. return;
  1734. }
  1735. if (sid != 0 &&
  1736. security_secid_to_secctx(sid, &secctx, &secctx_len) == 0)
  1737. audit_log_format(audit_buf, " subj=%s", secctx);
  1738. else
  1739. audit_log_task_context(audit_buf);
  1740. if (xp) {
  1741. family = xp->selector.family;
  1742. if (xp->security)
  1743. sctx = xp->security;
  1744. } else {
  1745. family = x->props.family;
  1746. if (x->security)
  1747. sctx = x->security;
  1748. }
  1749. if (sctx)
  1750. audit_log_format(audit_buf,
  1751. " sec_alg=%u sec_doi=%u sec_obj=%s",
  1752. sctx->ctx_alg, sctx->ctx_doi, sctx->ctx_str);
  1753. switch(family) {
  1754. case AF_INET:
  1755. {
  1756. struct in_addr saddr, daddr;
  1757. if (xp) {
  1758. saddr.s_addr = xp->selector.saddr.a4;
  1759. daddr.s_addr = xp->selector.daddr.a4;
  1760. } else {
  1761. saddr.s_addr = x->props.saddr.a4;
  1762. daddr.s_addr = x->id.daddr.a4;
  1763. }
  1764. audit_log_format(audit_buf,
  1765. " src=%u.%u.%u.%u dst=%u.%u.%u.%u",
  1766. NIPQUAD(saddr), NIPQUAD(daddr));
  1767. }
  1768. break;
  1769. case AF_INET6:
  1770. {
  1771. struct in6_addr saddr6, daddr6;
  1772. if (xp) {
  1773. memcpy(&saddr6, xp->selector.saddr.a6,
  1774. sizeof(struct in6_addr));
  1775. memcpy(&daddr6, xp->selector.daddr.a6,
  1776. sizeof(struct in6_addr));
  1777. } else {
  1778. memcpy(&saddr6, x->props.saddr.a6,
  1779. sizeof(struct in6_addr));
  1780. memcpy(&daddr6, x->id.daddr.a6,
  1781. sizeof(struct in6_addr));
  1782. }
  1783. audit_log_format(audit_buf,
  1784. " src=" NIP6_FMT " dst=" NIP6_FMT,
  1785. NIP6(saddr6), NIP6(daddr6));
  1786. }
  1787. break;
  1788. }
  1789. if (x)
  1790. audit_log_format(audit_buf, " spi=%lu(0x%lx) protocol=%s",
  1791. (unsigned long)ntohl(x->id.spi),
  1792. (unsigned long)ntohl(x->id.spi),
  1793. x->id.proto == IPPROTO_AH ? "AH" :
  1794. (x->id.proto == IPPROTO_ESP ?
  1795. "ESP" : "IPCOMP"));
  1796. audit_log_format(audit_buf, " res=%u", result);
  1797. audit_log_end(audit_buf);
  1798. }
  1799. EXPORT_SYMBOL(xfrm_audit_log);
  1800. #endif /* CONFIG_AUDITSYSCALL */
  1801. int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
  1802. {
  1803. int err = 0;
  1804. if (unlikely(afinfo == NULL))
  1805. return -EINVAL;
  1806. if (unlikely(afinfo->family >= NPROTO))
  1807. return -EAFNOSUPPORT;
  1808. write_lock_bh(&xfrm_policy_afinfo_lock);
  1809. if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
  1810. err = -ENOBUFS;
  1811. else {
  1812. struct dst_ops *dst_ops = afinfo->dst_ops;
  1813. if (likely(dst_ops->kmem_cachep == NULL))
  1814. dst_ops->kmem_cachep = xfrm_dst_cache;
  1815. if (likely(dst_ops->check == NULL))
  1816. dst_ops->check = xfrm_dst_check;
  1817. if (likely(dst_ops->negative_advice == NULL))
  1818. dst_ops->negative_advice = xfrm_negative_advice;
  1819. if (likely(dst_ops->link_failure == NULL))
  1820. dst_ops->link_failure = xfrm_link_failure;
  1821. if (likely(afinfo->garbage_collect == NULL))
  1822. afinfo->garbage_collect = __xfrm_garbage_collect;
  1823. xfrm_policy_afinfo[afinfo->family] = afinfo;
  1824. }
  1825. write_unlock_bh(&xfrm_policy_afinfo_lock);
  1826. return err;
  1827. }
  1828. EXPORT_SYMBOL(xfrm_policy_register_afinfo);
  1829. int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
  1830. {
  1831. int err = 0;
  1832. if (unlikely(afinfo == NULL))
  1833. return -EINVAL;
  1834. if (unlikely(afinfo->family >= NPROTO))
  1835. return -EAFNOSUPPORT;
  1836. write_lock_bh(&xfrm_policy_afinfo_lock);
  1837. if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
  1838. if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
  1839. err = -EINVAL;
  1840. else {
  1841. struct dst_ops *dst_ops = afinfo->dst_ops;
  1842. xfrm_policy_afinfo[afinfo->family] = NULL;
  1843. dst_ops->kmem_cachep = NULL;
  1844. dst_ops->check = NULL;
  1845. dst_ops->negative_advice = NULL;
  1846. dst_ops->link_failure = NULL;
  1847. afinfo->garbage_collect = NULL;
  1848. }
  1849. }
  1850. write_unlock_bh(&xfrm_policy_afinfo_lock);
  1851. return err;
  1852. }
  1853. EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
  1854. static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
  1855. {
  1856. struct xfrm_policy_afinfo *afinfo;
  1857. if (unlikely(family >= NPROTO))
  1858. return NULL;
  1859. read_lock(&xfrm_policy_afinfo_lock);
  1860. afinfo = xfrm_policy_afinfo[family];
  1861. if (unlikely(!afinfo))
  1862. read_unlock(&xfrm_policy_afinfo_lock);
  1863. return afinfo;
  1864. }
  1865. static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
  1866. {
  1867. read_unlock(&xfrm_policy_afinfo_lock);
  1868. }
  1869. static struct xfrm_policy_afinfo *xfrm_policy_lock_afinfo(unsigned int family)
  1870. {
  1871. struct xfrm_policy_afinfo *afinfo;
  1872. if (unlikely(family >= NPROTO))
  1873. return NULL;
  1874. write_lock_bh(&xfrm_policy_afinfo_lock);
  1875. afinfo = xfrm_policy_afinfo[family];
  1876. if (unlikely(!afinfo))
  1877. write_unlock_bh(&xfrm_policy_afinfo_lock);
  1878. return afinfo;
  1879. }
  1880. static void xfrm_policy_unlock_afinfo(struct xfrm_policy_afinfo *afinfo)
  1881. {
  1882. write_unlock_bh(&xfrm_policy_afinfo_lock);
  1883. }
  1884. static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
  1885. {
  1886. switch (event) {
  1887. case NETDEV_DOWN:
  1888. xfrm_flush_bundles();
  1889. }
  1890. return NOTIFY_DONE;
  1891. }
  1892. static struct notifier_block xfrm_dev_notifier = {
  1893. xfrm_dev_event,
  1894. NULL,
  1895. 0
  1896. };
  1897. static void __init xfrm_policy_init(void)
  1898. {
  1899. unsigned int hmask, sz;
  1900. int dir;
  1901. xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
  1902. sizeof(struct xfrm_dst),
  1903. 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  1904. NULL, NULL);
  1905. hmask = 8 - 1;
  1906. sz = (hmask+1) * sizeof(struct hlist_head);
  1907. xfrm_policy_byidx = xfrm_hash_alloc(sz);
  1908. xfrm_idx_hmask = hmask;
  1909. if (!xfrm_policy_byidx)
  1910. panic("XFRM: failed to allocate byidx hash\n");
  1911. for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
  1912. struct xfrm_policy_hash *htab;
  1913. INIT_HLIST_HEAD(&xfrm_policy_inexact[dir]);
  1914. htab = &xfrm_policy_bydst[dir];
  1915. htab->table = xfrm_hash_alloc(sz);
  1916. htab->hmask = hmask;
  1917. if (!htab->table)
  1918. panic("XFRM: failed to allocate bydst hash\n");
  1919. }
  1920. INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task);
  1921. register_netdevice_notifier(&xfrm_dev_notifier);
  1922. }
  1923. void __init xfrm_init(void)
  1924. {
  1925. xfrm_state_init();
  1926. xfrm_policy_init();
  1927. xfrm_input_init();
  1928. }
  1929. #ifdef CONFIG_XFRM_MIGRATE
  1930. static int xfrm_migrate_selector_match(struct xfrm_selector *sel_cmp,
  1931. struct xfrm_selector *sel_tgt)
  1932. {
  1933. if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
  1934. if (sel_tgt->family == sel_cmp->family &&
  1935. xfrm_addr_cmp(&sel_tgt->daddr, &sel_cmp->daddr,
  1936. sel_cmp->family) == 0 &&
  1937. xfrm_addr_cmp(&sel_tgt->saddr, &sel_cmp->saddr,
  1938. sel_cmp->family) == 0 &&
  1939. sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
  1940. sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
  1941. return 1;
  1942. }
  1943. } else {
  1944. if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
  1945. return 1;
  1946. }
  1947. }
  1948. return 0;
  1949. }
  1950. static struct xfrm_policy * xfrm_migrate_policy_find(struct xfrm_selector *sel,
  1951. u8 dir, u8 type)
  1952. {
  1953. struct xfrm_policy *pol, *ret = NULL;
  1954. struct hlist_node *entry;
  1955. struct hlist_head *chain;
  1956. u32 priority = ~0U;
  1957. read_lock_bh(&xfrm_policy_lock);
  1958. chain = policy_hash_direct(&sel->daddr, &sel->saddr, sel->family, dir);
  1959. hlist_for_each_entry(pol, entry, chain, bydst) {
  1960. if (xfrm_migrate_selector_match(sel, &pol->selector) &&
  1961. pol->type == type) {
  1962. ret = pol;
  1963. priority = ret->priority;
  1964. break;
  1965. }
  1966. }
  1967. chain = &xfrm_policy_inexact[dir];
  1968. hlist_for_each_entry(pol, entry, chain, bydst) {
  1969. if (xfrm_migrate_selector_match(sel, &pol->selector) &&
  1970. pol->type == type &&
  1971. pol->priority < priority) {
  1972. ret = pol;
  1973. break;
  1974. }
  1975. }
  1976. if (ret)
  1977. xfrm_pol_hold(ret);
  1978. read_unlock_bh(&xfrm_policy_lock);
  1979. return ret;
  1980. }
  1981. static int migrate_tmpl_match(struct xfrm_migrate *m, struct xfrm_tmpl *t)
  1982. {
  1983. int match = 0;
  1984. if (t->mode == m->mode && t->id.proto == m->proto &&
  1985. (m->reqid == 0 || t->reqid == m->reqid)) {
  1986. switch (t->mode) {
  1987. case XFRM_MODE_TUNNEL:
  1988. case XFRM_MODE_BEET:
  1989. if (xfrm_addr_cmp(&t->id.daddr, &m->old_daddr,
  1990. m->old_family) == 0 &&
  1991. xfrm_addr_cmp(&t->saddr, &m->old_saddr,
  1992. m->old_family) == 0) {
  1993. match = 1;
  1994. }
  1995. break;
  1996. case XFRM_MODE_TRANSPORT:
  1997. /* in case of transport mode, template does not store
  1998. any IP addresses, hence we just compare mode and
  1999. protocol */
  2000. match = 1;
  2001. break;
  2002. default:
  2003. break;
  2004. }
  2005. }
  2006. return match;
  2007. }
  2008. /* update endpoint address(es) of template(s) */
  2009. static int xfrm_policy_migrate(struct xfrm_policy *pol,
  2010. struct xfrm_migrate *m, int num_migrate)
  2011. {
  2012. struct xfrm_migrate *mp;
  2013. struct dst_entry *dst;
  2014. int i, j, n = 0;
  2015. write_lock_bh(&pol->lock);
  2016. if (unlikely(pol->dead)) {
  2017. /* target policy has been deleted */
  2018. write_unlock_bh(&pol->lock);
  2019. return -ENOENT;
  2020. }
  2021. for (i = 0; i < pol->xfrm_nr; i++) {
  2022. for (j = 0, mp = m; j < num_migrate; j++, mp++) {
  2023. if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
  2024. continue;
  2025. n++;
  2026. if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL)
  2027. continue;
  2028. /* update endpoints */
  2029. memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
  2030. sizeof(pol->xfrm_vec[i].id.daddr));
  2031. memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
  2032. sizeof(pol->xfrm_vec[i].saddr));
  2033. pol->xfrm_vec[i].encap_family = mp->new_family;
  2034. /* flush bundles */
  2035. while ((dst = pol->bundles) != NULL) {
  2036. pol->bundles = dst->next;
  2037. dst_free(dst);
  2038. }
  2039. }
  2040. }
  2041. write_unlock_bh(&pol->lock);
  2042. if (!n)
  2043. return -ENODATA;
  2044. return 0;
  2045. }
  2046. static int xfrm_migrate_check(struct xfrm_migrate *m, int num_migrate)
  2047. {
  2048. int i, j;
  2049. if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
  2050. return -EINVAL;
  2051. for (i = 0; i < num_migrate; i++) {
  2052. if ((xfrm_addr_cmp(&m[i].old_daddr, &m[i].new_daddr,
  2053. m[i].old_family) == 0) &&
  2054. (xfrm_addr_cmp(&m[i].old_saddr, &m[i].new_saddr,
  2055. m[i].old_family) == 0))
  2056. return -EINVAL;
  2057. if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
  2058. xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
  2059. return -EINVAL;
  2060. /* check if there is any duplicated entry */
  2061. for (j = i + 1; j < num_migrate; j++) {
  2062. if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
  2063. sizeof(m[i].old_daddr)) &&
  2064. !memcmp(&m[i].old_saddr, &m[j].old_saddr,
  2065. sizeof(m[i].old_saddr)) &&
  2066. m[i].proto == m[j].proto &&
  2067. m[i].mode == m[j].mode &&
  2068. m[i].reqid == m[j].reqid &&
  2069. m[i].old_family == m[j].old_family)
  2070. return -EINVAL;
  2071. }
  2072. }
  2073. return 0;
  2074. }
  2075. int xfrm_migrate(struct xfrm_selector *sel, u8 dir, u8 type,
  2076. struct xfrm_migrate *m, int num_migrate)
  2077. {
  2078. int i, err, nx_cur = 0, nx_new = 0;
  2079. struct xfrm_policy *pol = NULL;
  2080. struct xfrm_state *x, *xc;
  2081. struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
  2082. struct xfrm_state *x_new[XFRM_MAX_DEPTH];
  2083. struct xfrm_migrate *mp;
  2084. if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
  2085. goto out;
  2086. /* Stage 1 - find policy */
  2087. if ((pol = xfrm_migrate_policy_find(sel, dir, type)) == NULL) {
  2088. err = -ENOENT;
  2089. goto out;
  2090. }
  2091. /* Stage 2 - find and update state(s) */
  2092. for (i = 0, mp = m; i < num_migrate; i++, mp++) {
  2093. if ((x = xfrm_migrate_state_find(mp))) {
  2094. x_cur[nx_cur] = x;
  2095. nx_cur++;
  2096. if ((xc = xfrm_state_migrate(x, mp))) {
  2097. x_new[nx_new] = xc;
  2098. nx_new++;
  2099. } else {
  2100. err = -ENODATA;
  2101. goto restore_state;
  2102. }
  2103. }
  2104. }
  2105. /* Stage 3 - update policy */
  2106. if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
  2107. goto restore_state;
  2108. /* Stage 4 - delete old state(s) */
  2109. if (nx_cur) {
  2110. xfrm_states_put(x_cur, nx_cur);
  2111. xfrm_states_delete(x_cur, nx_cur);
  2112. }
  2113. /* Stage 5 - announce */
  2114. km_migrate(sel, dir, type, m, num_migrate);
  2115. xfrm_pol_put(pol);
  2116. return 0;
  2117. out:
  2118. return err;
  2119. restore_state:
  2120. if (pol)
  2121. xfrm_pol_put(pol);
  2122. if (nx_cur)
  2123. xfrm_states_put(x_cur, nx_cur);
  2124. if (nx_new)
  2125. xfrm_states_delete(x_new, nx_new);
  2126. return err;
  2127. }
  2128. EXPORT_SYMBOL(xfrm_migrate);
  2129. #endif