cache.c 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674
  1. /*
  2. * Copyright (c) 2004 Topspin Communications. All rights reserved.
  3. * Copyright (c) 2005 Intel Corporation. All rights reserved.
  4. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
  5. * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
  6. *
  7. * This software is available to you under a choice of one of two
  8. * licenses. You may choose to be licensed under the terms of the GNU
  9. * General Public License (GPL) Version 2, available from the file
  10. * COPYING in the main directory of this source tree, or the
  11. * OpenIB.org BSD license below:
  12. *
  13. * Redistribution and use in source and binary forms, with or
  14. * without modification, are permitted provided that the following
  15. * conditions are met:
  16. *
  17. * - Redistributions of source code must retain the above
  18. * copyright notice, this list of conditions and the following
  19. * disclaimer.
  20. *
  21. * - Redistributions in binary form must reproduce the above
  22. * copyright notice, this list of conditions and the following
  23. * disclaimer in the documentation and/or other materials
  24. * provided with the distribution.
  25. *
  26. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33. * SOFTWARE.
  34. */
  35. #include <linux/module.h>
  36. #include <linux/errno.h>
  37. #include <linux/slab.h>
  38. #include <linux/workqueue.h>
  39. #include <linux/netdevice.h>
  40. #include <net/addrconf.h>
  41. #include <rdma/ib_cache.h>
  42. #include "core_priv.h"
  43. struct ib_pkey_cache {
  44. int table_len;
  45. u16 table[];
  46. };
  47. struct ib_update_work {
  48. struct work_struct work;
  49. struct ib_event event;
  50. bool enforce_security;
  51. };
  52. union ib_gid zgid;
  53. EXPORT_SYMBOL(zgid);
  54. enum gid_attr_find_mask {
  55. GID_ATTR_FIND_MASK_GID = 1UL << 0,
  56. GID_ATTR_FIND_MASK_NETDEV = 1UL << 1,
  57. GID_ATTR_FIND_MASK_DEFAULT = 1UL << 2,
  58. GID_ATTR_FIND_MASK_GID_TYPE = 1UL << 3,
  59. };
  60. enum gid_table_entry_state {
  61. GID_TABLE_ENTRY_INVALID = 1,
  62. GID_TABLE_ENTRY_VALID = 2,
  63. /*
  64. * Indicates that entry is pending to be removed, there may
  65. * be active users of this GID entry.
  66. * When last user of the GID entry releases reference to it,
  67. * GID entry is detached from the table.
  68. */
  69. GID_TABLE_ENTRY_PENDING_DEL = 3,
  70. };
  71. struct roce_gid_ndev_storage {
  72. struct rcu_head rcu_head;
  73. struct net_device *ndev;
  74. };
  75. struct ib_gid_table_entry {
  76. struct kref kref;
  77. struct work_struct del_work;
  78. struct ib_gid_attr attr;
  79. void *context;
  80. /* Store the ndev pointer to release reference later on in
  81. * call_rcu context because by that time gid_table_entry
  82. * and attr might be already freed. So keep a copy of it.
  83. * ndev_storage is freed by rcu callback.
  84. */
  85. struct roce_gid_ndev_storage *ndev_storage;
  86. enum gid_table_entry_state state;
  87. };
  88. struct ib_gid_table {
  89. int sz;
  90. /* In RoCE, adding a GID to the table requires:
  91. * (a) Find if this GID is already exists.
  92. * (b) Find a free space.
  93. * (c) Write the new GID
  94. *
  95. * Delete requires different set of operations:
  96. * (a) Find the GID
  97. * (b) Delete it.
  98. *
  99. **/
  100. /* Any writer to data_vec must hold this lock and the write side of
  101. * rwlock. Readers must hold only rwlock. All writers must be in a
  102. * sleepable context.
  103. */
  104. struct mutex lock;
  105. /* rwlock protects data_vec[ix]->state and entry pointer.
  106. */
  107. rwlock_t rwlock;
  108. struct ib_gid_table_entry **data_vec;
  109. /* bit field, each bit indicates the index of default GID */
  110. u32 default_gid_indices;
  111. };
  112. static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port)
  113. {
  114. struct ib_event event;
  115. event.device = ib_dev;
  116. event.element.port_num = port;
  117. event.event = IB_EVENT_GID_CHANGE;
  118. ib_dispatch_event_clients(&event);
  119. }
  120. static const char * const gid_type_str[] = {
  121. /* IB/RoCE v1 value is set for IB_GID_TYPE_IB and IB_GID_TYPE_ROCE for
  122. * user space compatibility reasons.
  123. */
  124. [IB_GID_TYPE_IB] = "IB/RoCE v1",
  125. [IB_GID_TYPE_ROCE] = "IB/RoCE v1",
  126. [IB_GID_TYPE_ROCE_UDP_ENCAP] = "RoCE v2",
  127. };
  128. const char *ib_cache_gid_type_str(enum ib_gid_type gid_type)
  129. {
  130. if (gid_type < ARRAY_SIZE(gid_type_str) && gid_type_str[gid_type])
  131. return gid_type_str[gid_type];
  132. return "Invalid GID type";
  133. }
  134. EXPORT_SYMBOL(ib_cache_gid_type_str);
  135. /** rdma_is_zero_gid - Check if given GID is zero or not.
  136. * @gid: GID to check
  137. * Returns true if given GID is zero, returns false otherwise.
  138. */
  139. bool rdma_is_zero_gid(const union ib_gid *gid)
  140. {
  141. return !memcmp(gid, &zgid, sizeof(*gid));
  142. }
  143. EXPORT_SYMBOL(rdma_is_zero_gid);
  144. /** is_gid_index_default - Check if a given index belongs to
  145. * reserved default GIDs or not.
  146. * @table: GID table pointer
  147. * @index: Index to check in GID table
  148. * Returns true if index is one of the reserved default GID index otherwise
  149. * returns false.
  150. */
  151. static bool is_gid_index_default(const struct ib_gid_table *table,
  152. unsigned int index)
  153. {
  154. return index < 32 && (BIT(index) & table->default_gid_indices);
  155. }
  156. int ib_cache_gid_parse_type_str(const char *buf)
  157. {
  158. unsigned int i;
  159. size_t len;
  160. int err = -EINVAL;
  161. len = strlen(buf);
  162. if (len == 0)
  163. return -EINVAL;
  164. if (buf[len - 1] == '\n')
  165. len--;
  166. for (i = 0; i < ARRAY_SIZE(gid_type_str); ++i)
  167. if (gid_type_str[i] && !strncmp(buf, gid_type_str[i], len) &&
  168. len == strlen(gid_type_str[i])) {
  169. err = i;
  170. break;
  171. }
  172. return err;
  173. }
  174. EXPORT_SYMBOL(ib_cache_gid_parse_type_str);
  175. static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u8 port)
  176. {
  177. return device->port_data[port].cache.gid;
  178. }
  179. static bool is_gid_entry_free(const struct ib_gid_table_entry *entry)
  180. {
  181. return !entry;
  182. }
  183. static bool is_gid_entry_valid(const struct ib_gid_table_entry *entry)
  184. {
  185. return entry && entry->state == GID_TABLE_ENTRY_VALID;
  186. }
  187. static void schedule_free_gid(struct kref *kref)
  188. {
  189. struct ib_gid_table_entry *entry =
  190. container_of(kref, struct ib_gid_table_entry, kref);
  191. queue_work(ib_wq, &entry->del_work);
  192. }
  193. static void put_gid_ndev(struct rcu_head *head)
  194. {
  195. struct roce_gid_ndev_storage *storage =
  196. container_of(head, struct roce_gid_ndev_storage, rcu_head);
  197. WARN_ON(!storage->ndev);
  198. /* At this point its safe to release netdev reference,
  199. * as all callers working on gid_attr->ndev are done
  200. * using this netdev.
  201. */
  202. dev_put(storage->ndev);
  203. kfree(storage);
  204. }
  205. static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
  206. {
  207. struct ib_device *device = entry->attr.device;
  208. u8 port_num = entry->attr.port_num;
  209. struct ib_gid_table *table = rdma_gid_table(device, port_num);
  210. dev_dbg(&device->dev, "%s port=%d index=%d gid %pI6\n", __func__,
  211. port_num, entry->attr.index, entry->attr.gid.raw);
  212. write_lock_irq(&table->rwlock);
  213. /*
  214. * The only way to avoid overwriting NULL in table is
  215. * by comparing if it is same entry in table or not!
  216. * If new entry in table is added by the time we free here,
  217. * don't overwrite the table entry.
  218. */
  219. if (entry == table->data_vec[entry->attr.index])
  220. table->data_vec[entry->attr.index] = NULL;
  221. /* Now this index is ready to be allocated */
  222. write_unlock_irq(&table->rwlock);
  223. if (entry->ndev_storage)
  224. call_rcu(&entry->ndev_storage->rcu_head, put_gid_ndev);
  225. kfree(entry);
  226. }
  227. static void free_gid_entry(struct kref *kref)
  228. {
  229. struct ib_gid_table_entry *entry =
  230. container_of(kref, struct ib_gid_table_entry, kref);
  231. free_gid_entry_locked(entry);
  232. }
  233. /**
  234. * free_gid_work - Release reference to the GID entry
  235. * @work: Work structure to refer to GID entry which needs to be
  236. * deleted.
  237. *
  238. * free_gid_work() frees the entry from the HCA's hardware table
  239. * if provider supports it. It releases reference to netdevice.
  240. */
  241. static void free_gid_work(struct work_struct *work)
  242. {
  243. struct ib_gid_table_entry *entry =
  244. container_of(work, struct ib_gid_table_entry, del_work);
  245. struct ib_device *device = entry->attr.device;
  246. u8 port_num = entry->attr.port_num;
  247. struct ib_gid_table *table = rdma_gid_table(device, port_num);
  248. mutex_lock(&table->lock);
  249. free_gid_entry_locked(entry);
  250. mutex_unlock(&table->lock);
  251. }
  252. static struct ib_gid_table_entry *
  253. alloc_gid_entry(const struct ib_gid_attr *attr)
  254. {
  255. struct ib_gid_table_entry *entry;
  256. struct net_device *ndev;
  257. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  258. if (!entry)
  259. return NULL;
  260. ndev = rcu_dereference_protected(attr->ndev, 1);
  261. if (ndev) {
  262. entry->ndev_storage = kzalloc(sizeof(*entry->ndev_storage),
  263. GFP_KERNEL);
  264. if (!entry->ndev_storage) {
  265. kfree(entry);
  266. return NULL;
  267. }
  268. dev_hold(ndev);
  269. entry->ndev_storage->ndev = ndev;
  270. }
  271. kref_init(&entry->kref);
  272. memcpy(&entry->attr, attr, sizeof(*attr));
  273. INIT_WORK(&entry->del_work, free_gid_work);
  274. entry->state = GID_TABLE_ENTRY_INVALID;
  275. return entry;
  276. }
  277. static void store_gid_entry(struct ib_gid_table *table,
  278. struct ib_gid_table_entry *entry)
  279. {
  280. entry->state = GID_TABLE_ENTRY_VALID;
  281. dev_dbg(&entry->attr.device->dev, "%s port=%d index=%d gid %pI6\n",
  282. __func__, entry->attr.port_num, entry->attr.index,
  283. entry->attr.gid.raw);
  284. lockdep_assert_held(&table->lock);
  285. write_lock_irq(&table->rwlock);
  286. table->data_vec[entry->attr.index] = entry;
  287. write_unlock_irq(&table->rwlock);
  288. }
  289. static void get_gid_entry(struct ib_gid_table_entry *entry)
  290. {
  291. kref_get(&entry->kref);
  292. }
  293. static void put_gid_entry(struct ib_gid_table_entry *entry)
  294. {
  295. kref_put(&entry->kref, schedule_free_gid);
  296. }
  297. static void put_gid_entry_locked(struct ib_gid_table_entry *entry)
  298. {
  299. kref_put(&entry->kref, free_gid_entry);
  300. }
  301. static int add_roce_gid(struct ib_gid_table_entry *entry)
  302. {
  303. const struct ib_gid_attr *attr = &entry->attr;
  304. int ret;
  305. if (!attr->ndev) {
  306. dev_err(&attr->device->dev, "%s NULL netdev port=%d index=%d\n",
  307. __func__, attr->port_num, attr->index);
  308. return -EINVAL;
  309. }
  310. if (rdma_cap_roce_gid_table(attr->device, attr->port_num)) {
  311. ret = attr->device->ops.add_gid(attr, &entry->context);
  312. if (ret) {
  313. dev_err(&attr->device->dev,
  314. "%s GID add failed port=%d index=%d\n",
  315. __func__, attr->port_num, attr->index);
  316. return ret;
  317. }
  318. }
  319. return 0;
  320. }
  321. /**
  322. * del_gid - Delete GID table entry
  323. *
  324. * @ib_dev: IB device whose GID entry to be deleted
  325. * @port: Port number of the IB device
  326. * @table: GID table of the IB device for a port
  327. * @ix: GID entry index to delete
  328. *
  329. */
  330. static void del_gid(struct ib_device *ib_dev, u8 port,
  331. struct ib_gid_table *table, int ix)
  332. {
  333. struct roce_gid_ndev_storage *ndev_storage;
  334. struct ib_gid_table_entry *entry;
  335. lockdep_assert_held(&table->lock);
  336. dev_dbg(&ib_dev->dev, "%s port=%d index=%d gid %pI6\n", __func__, port,
  337. ix, table->data_vec[ix]->attr.gid.raw);
  338. write_lock_irq(&table->rwlock);
  339. entry = table->data_vec[ix];
  340. entry->state = GID_TABLE_ENTRY_PENDING_DEL;
  341. /*
  342. * For non RoCE protocol, GID entry slot is ready to use.
  343. */
  344. if (!rdma_protocol_roce(ib_dev, port))
  345. table->data_vec[ix] = NULL;
  346. write_unlock_irq(&table->rwlock);
  347. ndev_storage = entry->ndev_storage;
  348. if (ndev_storage) {
  349. entry->ndev_storage = NULL;
  350. rcu_assign_pointer(entry->attr.ndev, NULL);
  351. call_rcu(&ndev_storage->rcu_head, put_gid_ndev);
  352. }
  353. if (rdma_cap_roce_gid_table(ib_dev, port))
  354. ib_dev->ops.del_gid(&entry->attr, &entry->context);
  355. put_gid_entry_locked(entry);
  356. }
  357. /**
  358. * add_modify_gid - Add or modify GID table entry
  359. *
  360. * @table: GID table in which GID to be added or modified
  361. * @attr: Attributes of the GID
  362. *
  363. * Returns 0 on success or appropriate error code. It accepts zero
  364. * GID addition for non RoCE ports for HCA's who report them as valid
  365. * GID. However such zero GIDs are not added to the cache.
  366. */
  367. static int add_modify_gid(struct ib_gid_table *table,
  368. const struct ib_gid_attr *attr)
  369. {
  370. struct ib_gid_table_entry *entry;
  371. int ret = 0;
  372. /*
  373. * Invalidate any old entry in the table to make it safe to write to
  374. * this index.
  375. */
  376. if (is_gid_entry_valid(table->data_vec[attr->index]))
  377. del_gid(attr->device, attr->port_num, table, attr->index);
  378. /*
  379. * Some HCA's report multiple GID entries with only one valid GID, and
  380. * leave other unused entries as the zero GID. Convert zero GIDs to
  381. * empty table entries instead of storing them.
  382. */
  383. if (rdma_is_zero_gid(&attr->gid))
  384. return 0;
  385. entry = alloc_gid_entry(attr);
  386. if (!entry)
  387. return -ENOMEM;
  388. if (rdma_protocol_roce(attr->device, attr->port_num)) {
  389. ret = add_roce_gid(entry);
  390. if (ret)
  391. goto done;
  392. }
  393. store_gid_entry(table, entry);
  394. return 0;
  395. done:
  396. put_gid_entry(entry);
  397. return ret;
  398. }
  399. /* rwlock should be read locked, or lock should be held */
  400. static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,
  401. const struct ib_gid_attr *val, bool default_gid,
  402. unsigned long mask, int *pempty)
  403. {
  404. int i = 0;
  405. int found = -1;
  406. int empty = pempty ? -1 : 0;
  407. while (i < table->sz && (found < 0 || empty < 0)) {
  408. struct ib_gid_table_entry *data = table->data_vec[i];
  409. struct ib_gid_attr *attr;
  410. int curr_index = i;
  411. i++;
  412. /* find_gid() is used during GID addition where it is expected
  413. * to return a free entry slot which is not duplicate.
  414. * Free entry slot is requested and returned if pempty is set,
  415. * so lookup free slot only if requested.
  416. */
  417. if (pempty && empty < 0) {
  418. if (is_gid_entry_free(data) &&
  419. default_gid ==
  420. is_gid_index_default(table, curr_index)) {
  421. /*
  422. * Found an invalid (free) entry; allocate it.
  423. * If default GID is requested, then our
  424. * found slot must be one of the DEFAULT
  425. * reserved slots or we fail.
  426. * This ensures that only DEFAULT reserved
  427. * slots are used for default property GIDs.
  428. */
  429. empty = curr_index;
  430. }
  431. }
  432. /*
  433. * Additionally find_gid() is used to find valid entry during
  434. * lookup operation; so ignore the entries which are marked as
  435. * pending for removal and the entries which are marked as
  436. * invalid.
  437. */
  438. if (!is_gid_entry_valid(data))
  439. continue;
  440. if (found >= 0)
  441. continue;
  442. attr = &data->attr;
  443. if (mask & GID_ATTR_FIND_MASK_GID_TYPE &&
  444. attr->gid_type != val->gid_type)
  445. continue;
  446. if (mask & GID_ATTR_FIND_MASK_GID &&
  447. memcmp(gid, &data->attr.gid, sizeof(*gid)))
  448. continue;
  449. if (mask & GID_ATTR_FIND_MASK_NETDEV &&
  450. attr->ndev != val->ndev)
  451. continue;
  452. if (mask & GID_ATTR_FIND_MASK_DEFAULT &&
  453. is_gid_index_default(table, curr_index) != default_gid)
  454. continue;
  455. found = curr_index;
  456. }
  457. if (pempty)
  458. *pempty = empty;
  459. return found;
  460. }
  461. static void make_default_gid(struct net_device *dev, union ib_gid *gid)
  462. {
  463. gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
  464. addrconf_ifid_eui48(&gid->raw[8], dev);
  465. }
  466. static int __ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
  467. union ib_gid *gid, struct ib_gid_attr *attr,
  468. unsigned long mask, bool default_gid)
  469. {
  470. struct ib_gid_table *table;
  471. int ret = 0;
  472. int empty;
  473. int ix;
  474. /* Do not allow adding zero GID in support of
  475. * IB spec version 1.3 section 4.1.1 point (6) and
  476. * section 12.7.10 and section 12.7.20
  477. */
  478. if (rdma_is_zero_gid(gid))
  479. return -EINVAL;
  480. table = rdma_gid_table(ib_dev, port);
  481. mutex_lock(&table->lock);
  482. ix = find_gid(table, gid, attr, default_gid, mask, &empty);
  483. if (ix >= 0)
  484. goto out_unlock;
  485. if (empty < 0) {
  486. ret = -ENOSPC;
  487. goto out_unlock;
  488. }
  489. attr->device = ib_dev;
  490. attr->index = empty;
  491. attr->port_num = port;
  492. attr->gid = *gid;
  493. ret = add_modify_gid(table, attr);
  494. if (!ret)
  495. dispatch_gid_change_event(ib_dev, port);
  496. out_unlock:
  497. mutex_unlock(&table->lock);
  498. if (ret)
  499. pr_warn("%s: unable to add gid %pI6 error=%d\n",
  500. __func__, gid->raw, ret);
  501. return ret;
  502. }
  503. int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
  504. union ib_gid *gid, struct ib_gid_attr *attr)
  505. {
  506. unsigned long mask = GID_ATTR_FIND_MASK_GID |
  507. GID_ATTR_FIND_MASK_GID_TYPE |
  508. GID_ATTR_FIND_MASK_NETDEV;
  509. return __ib_cache_gid_add(ib_dev, port, gid, attr, mask, false);
  510. }
  511. static int
  512. _ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
  513. union ib_gid *gid, struct ib_gid_attr *attr,
  514. unsigned long mask, bool default_gid)
  515. {
  516. struct ib_gid_table *table;
  517. int ret = 0;
  518. int ix;
  519. table = rdma_gid_table(ib_dev, port);
  520. mutex_lock(&table->lock);
  521. ix = find_gid(table, gid, attr, default_gid, mask, NULL);
  522. if (ix < 0) {
  523. ret = -EINVAL;
  524. goto out_unlock;
  525. }
  526. del_gid(ib_dev, port, table, ix);
  527. dispatch_gid_change_event(ib_dev, port);
  528. out_unlock:
  529. mutex_unlock(&table->lock);
  530. if (ret)
  531. pr_debug("%s: can't delete gid %pI6 error=%d\n",
  532. __func__, gid->raw, ret);
  533. return ret;
  534. }
  535. int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
  536. union ib_gid *gid, struct ib_gid_attr *attr)
  537. {
  538. unsigned long mask = GID_ATTR_FIND_MASK_GID |
  539. GID_ATTR_FIND_MASK_GID_TYPE |
  540. GID_ATTR_FIND_MASK_DEFAULT |
  541. GID_ATTR_FIND_MASK_NETDEV;
  542. return _ib_cache_gid_del(ib_dev, port, gid, attr, mask, false);
  543. }
  544. int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
  545. struct net_device *ndev)
  546. {
  547. struct ib_gid_table *table;
  548. int ix;
  549. bool deleted = false;
  550. table = rdma_gid_table(ib_dev, port);
  551. mutex_lock(&table->lock);
  552. for (ix = 0; ix < table->sz; ix++) {
  553. if (is_gid_entry_valid(table->data_vec[ix]) &&
  554. table->data_vec[ix]->attr.ndev == ndev) {
  555. del_gid(ib_dev, port, table, ix);
  556. deleted = true;
  557. }
  558. }
  559. mutex_unlock(&table->lock);
  560. if (deleted)
  561. dispatch_gid_change_event(ib_dev, port);
  562. return 0;
  563. }
  564. /**
  565. * rdma_find_gid_by_port - Returns the GID entry attributes when it finds
  566. * a valid GID entry for given search parameters. It searches for the specified
  567. * GID value in the local software cache.
  568. * @device: The device to query.
  569. * @gid: The GID value to search for.
  570. * @gid_type: The GID type to search for.
  571. * @port_num: The port number of the device where the GID value should be
  572. * searched.
  573. * @ndev: In RoCE, the net device of the device. NULL means ignore.
  574. *
  575. * Returns sgid attributes if the GID is found with valid reference or
  576. * returns ERR_PTR for the error.
  577. * The caller must invoke rdma_put_gid_attr() to release the reference.
  578. */
  579. const struct ib_gid_attr *
  580. rdma_find_gid_by_port(struct ib_device *ib_dev,
  581. const union ib_gid *gid,
  582. enum ib_gid_type gid_type,
  583. u8 port, struct net_device *ndev)
  584. {
  585. int local_index;
  586. struct ib_gid_table *table;
  587. unsigned long mask = GID_ATTR_FIND_MASK_GID |
  588. GID_ATTR_FIND_MASK_GID_TYPE;
  589. struct ib_gid_attr val = {.ndev = ndev, .gid_type = gid_type};
  590. const struct ib_gid_attr *attr;
  591. unsigned long flags;
  592. if (!rdma_is_port_valid(ib_dev, port))
  593. return ERR_PTR(-ENOENT);
  594. table = rdma_gid_table(ib_dev, port);
  595. if (ndev)
  596. mask |= GID_ATTR_FIND_MASK_NETDEV;
  597. read_lock_irqsave(&table->rwlock, flags);
  598. local_index = find_gid(table, gid, &val, false, mask, NULL);
  599. if (local_index >= 0) {
  600. get_gid_entry(table->data_vec[local_index]);
  601. attr = &table->data_vec[local_index]->attr;
  602. read_unlock_irqrestore(&table->rwlock, flags);
  603. return attr;
  604. }
  605. read_unlock_irqrestore(&table->rwlock, flags);
  606. return ERR_PTR(-ENOENT);
  607. }
  608. EXPORT_SYMBOL(rdma_find_gid_by_port);
  609. /**
  610. * rdma_find_gid_by_filter - Returns the GID table attribute where a
  611. * specified GID value occurs
  612. * @device: The device to query.
  613. * @gid: The GID value to search for.
  614. * @port: The port number of the device where the GID value could be
  615. * searched.
  616. * @filter: The filter function is executed on any matching GID in the table.
  617. * If the filter function returns true, the corresponding index is returned,
  618. * otherwise, we continue searching the GID table. It's guaranteed that
  619. * while filter is executed, ndev field is valid and the structure won't
  620. * change. filter is executed in an atomic context. filter must not be NULL.
  621. *
  622. * rdma_find_gid_by_filter() searches for the specified GID value
  623. * of which the filter function returns true in the port's GID table.
  624. *
  625. */
  626. const struct ib_gid_attr *rdma_find_gid_by_filter(
  627. struct ib_device *ib_dev, const union ib_gid *gid, u8 port,
  628. bool (*filter)(const union ib_gid *gid, const struct ib_gid_attr *,
  629. void *),
  630. void *context)
  631. {
  632. const struct ib_gid_attr *res = ERR_PTR(-ENOENT);
  633. struct ib_gid_table *table;
  634. unsigned long flags;
  635. unsigned int i;
  636. if (!rdma_is_port_valid(ib_dev, port))
  637. return ERR_PTR(-EINVAL);
  638. table = rdma_gid_table(ib_dev, port);
  639. read_lock_irqsave(&table->rwlock, flags);
  640. for (i = 0; i < table->sz; i++) {
  641. struct ib_gid_table_entry *entry = table->data_vec[i];
  642. if (!is_gid_entry_valid(entry))
  643. continue;
  644. if (memcmp(gid, &entry->attr.gid, sizeof(*gid)))
  645. continue;
  646. if (filter(gid, &entry->attr, context)) {
  647. get_gid_entry(entry);
  648. res = &entry->attr;
  649. break;
  650. }
  651. }
  652. read_unlock_irqrestore(&table->rwlock, flags);
  653. return res;
  654. }
  655. static struct ib_gid_table *alloc_gid_table(int sz)
  656. {
  657. struct ib_gid_table *table = kzalloc(sizeof(*table), GFP_KERNEL);
  658. if (!table)
  659. return NULL;
  660. table->data_vec = kcalloc(sz, sizeof(*table->data_vec), GFP_KERNEL);
  661. if (!table->data_vec)
  662. goto err_free_table;
  663. mutex_init(&table->lock);
  664. table->sz = sz;
  665. rwlock_init(&table->rwlock);
  666. return table;
  667. err_free_table:
  668. kfree(table);
  669. return NULL;
  670. }
  671. static void release_gid_table(struct ib_device *device,
  672. struct ib_gid_table *table)
  673. {
  674. bool leak = false;
  675. int i;
  676. if (!table)
  677. return;
  678. for (i = 0; i < table->sz; i++) {
  679. if (is_gid_entry_free(table->data_vec[i]))
  680. continue;
  681. if (kref_read(&table->data_vec[i]->kref) > 1) {
  682. dev_err(&device->dev,
  683. "GID entry ref leak for index %d ref=%d\n", i,
  684. kref_read(&table->data_vec[i]->kref));
  685. leak = true;
  686. }
  687. }
  688. if (leak)
  689. return;
  690. mutex_destroy(&table->lock);
  691. kfree(table->data_vec);
  692. kfree(table);
  693. }
  694. static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,
  695. struct ib_gid_table *table)
  696. {
  697. int i;
  698. if (!table)
  699. return;
  700. mutex_lock(&table->lock);
  701. for (i = 0; i < table->sz; ++i) {
  702. if (is_gid_entry_valid(table->data_vec[i]))
  703. del_gid(ib_dev, port, table, i);
  704. }
  705. mutex_unlock(&table->lock);
  706. }
  707. void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
  708. struct net_device *ndev,
  709. unsigned long gid_type_mask,
  710. enum ib_cache_gid_default_mode mode)
  711. {
  712. union ib_gid gid = { };
  713. struct ib_gid_attr gid_attr;
  714. unsigned int gid_type;
  715. unsigned long mask;
  716. mask = GID_ATTR_FIND_MASK_GID_TYPE |
  717. GID_ATTR_FIND_MASK_DEFAULT |
  718. GID_ATTR_FIND_MASK_NETDEV;
  719. memset(&gid_attr, 0, sizeof(gid_attr));
  720. gid_attr.ndev = ndev;
  721. for (gid_type = 0; gid_type < IB_GID_TYPE_SIZE; ++gid_type) {
  722. if (1UL << gid_type & ~gid_type_mask)
  723. continue;
  724. gid_attr.gid_type = gid_type;
  725. if (mode == IB_CACHE_GID_DEFAULT_MODE_SET) {
  726. make_default_gid(ndev, &gid);
  727. __ib_cache_gid_add(ib_dev, port, &gid,
  728. &gid_attr, mask, true);
  729. } else if (mode == IB_CACHE_GID_DEFAULT_MODE_DELETE) {
  730. _ib_cache_gid_del(ib_dev, port, &gid,
  731. &gid_attr, mask, true);
  732. }
  733. }
  734. }
  735. static void gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
  736. struct ib_gid_table *table)
  737. {
  738. unsigned int i;
  739. unsigned long roce_gid_type_mask;
  740. unsigned int num_default_gids;
  741. roce_gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
  742. num_default_gids = hweight_long(roce_gid_type_mask);
  743. /* Reserve starting indices for default GIDs */
  744. for (i = 0; i < num_default_gids && i < table->sz; i++)
  745. table->default_gid_indices |= BIT(i);
  746. }
  747. static void gid_table_release_one(struct ib_device *ib_dev)
  748. {
  749. unsigned int p;
  750. rdma_for_each_port (ib_dev, p) {
  751. release_gid_table(ib_dev, ib_dev->port_data[p].cache.gid);
  752. ib_dev->port_data[p].cache.gid = NULL;
  753. }
  754. }
  755. static int _gid_table_setup_one(struct ib_device *ib_dev)
  756. {
  757. struct ib_gid_table *table;
  758. unsigned int rdma_port;
  759. rdma_for_each_port (ib_dev, rdma_port) {
  760. table = alloc_gid_table(
  761. ib_dev->port_data[rdma_port].immutable.gid_tbl_len);
  762. if (!table)
  763. goto rollback_table_setup;
  764. gid_table_reserve_default(ib_dev, rdma_port, table);
  765. ib_dev->port_data[rdma_port].cache.gid = table;
  766. }
  767. return 0;
  768. rollback_table_setup:
  769. gid_table_release_one(ib_dev);
  770. return -ENOMEM;
  771. }
  772. static void gid_table_cleanup_one(struct ib_device *ib_dev)
  773. {
  774. unsigned int p;
  775. rdma_for_each_port (ib_dev, p)
  776. cleanup_gid_table_port(ib_dev, p,
  777. ib_dev->port_data[p].cache.gid);
  778. }
  779. static int gid_table_setup_one(struct ib_device *ib_dev)
  780. {
  781. int err;
  782. err = _gid_table_setup_one(ib_dev);
  783. if (err)
  784. return err;
  785. rdma_roce_rescan_device(ib_dev);
  786. return err;
  787. }
  788. /**
  789. * rdma_query_gid - Read the GID content from the GID software cache
  790. * @device: Device to query the GID
  791. * @port_num: Port number of the device
  792. * @index: Index of the GID table entry to read
  793. * @gid: Pointer to GID where to store the entry's GID
  794. *
  795. * rdma_query_gid() only reads the GID entry content for requested device,
  796. * port and index. It reads for IB, RoCE and iWarp link layers. It doesn't
  797. * hold any reference to the GID table entry in the HCA or software cache.
  798. *
  799. * Returns 0 on success or appropriate error code.
  800. *
  801. */
  802. int rdma_query_gid(struct ib_device *device, u8 port_num,
  803. int index, union ib_gid *gid)
  804. {
  805. struct ib_gid_table *table;
  806. unsigned long flags;
  807. int res = -EINVAL;
  808. if (!rdma_is_port_valid(device, port_num))
  809. return -EINVAL;
  810. table = rdma_gid_table(device, port_num);
  811. read_lock_irqsave(&table->rwlock, flags);
  812. if (index < 0 || index >= table->sz ||
  813. !is_gid_entry_valid(table->data_vec[index]))
  814. goto done;
  815. memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid));
  816. res = 0;
  817. done:
  818. read_unlock_irqrestore(&table->rwlock, flags);
  819. return res;
  820. }
  821. EXPORT_SYMBOL(rdma_query_gid);
  822. /**
  823. * rdma_read_gid_hw_context - Read the HW GID context from GID attribute
  824. * @attr: Potinter to the GID attribute
  825. *
  826. * rdma_read_gid_hw_context() reads the drivers GID HW context corresponding
  827. * to the SGID attr. Callers are required to already be holding the reference
  828. * to an existing GID entry.
  829. *
  830. * Returns the HW GID context
  831. *
  832. */
  833. void *rdma_read_gid_hw_context(const struct ib_gid_attr *attr)
  834. {
  835. return container_of(attr, struct ib_gid_table_entry, attr)->context;
  836. }
  837. EXPORT_SYMBOL(rdma_read_gid_hw_context);
  838. /**
  839. * rdma_find_gid - Returns SGID attributes if the matching GID is found.
  840. * @device: The device to query.
  841. * @gid: The GID value to search for.
  842. * @gid_type: The GID type to search for.
  843. * @ndev: In RoCE, the net device of the device. NULL means ignore.
  844. *
  845. * rdma_find_gid() searches for the specified GID value in the software cache.
  846. *
  847. * Returns GID attributes if a valid GID is found or returns ERR_PTR for the
  848. * error. The caller must invoke rdma_put_gid_attr() to release the reference.
  849. *
  850. */
  851. const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,
  852. const union ib_gid *gid,
  853. enum ib_gid_type gid_type,
  854. struct net_device *ndev)
  855. {
  856. unsigned long mask = GID_ATTR_FIND_MASK_GID |
  857. GID_ATTR_FIND_MASK_GID_TYPE;
  858. struct ib_gid_attr gid_attr_val = {.ndev = ndev, .gid_type = gid_type};
  859. unsigned int p;
  860. if (ndev)
  861. mask |= GID_ATTR_FIND_MASK_NETDEV;
  862. rdma_for_each_port(device, p) {
  863. struct ib_gid_table *table;
  864. unsigned long flags;
  865. int index;
  866. table = device->port_data[p].cache.gid;
  867. read_lock_irqsave(&table->rwlock, flags);
  868. index = find_gid(table, gid, &gid_attr_val, false, mask, NULL);
  869. if (index >= 0) {
  870. const struct ib_gid_attr *attr;
  871. get_gid_entry(table->data_vec[index]);
  872. attr = &table->data_vec[index]->attr;
  873. read_unlock_irqrestore(&table->rwlock, flags);
  874. return attr;
  875. }
  876. read_unlock_irqrestore(&table->rwlock, flags);
  877. }
  878. return ERR_PTR(-ENOENT);
  879. }
  880. EXPORT_SYMBOL(rdma_find_gid);
  881. int ib_get_cached_pkey(struct ib_device *device,
  882. u8 port_num,
  883. int index,
  884. u16 *pkey)
  885. {
  886. struct ib_pkey_cache *cache;
  887. unsigned long flags;
  888. int ret = 0;
  889. if (!rdma_is_port_valid(device, port_num))
  890. return -EINVAL;
  891. read_lock_irqsave(&device->cache_lock, flags);
  892. cache = device->port_data[port_num].cache.pkey;
  893. if (!cache || index < 0 || index >= cache->table_len)
  894. ret = -EINVAL;
  895. else
  896. *pkey = cache->table[index];
  897. read_unlock_irqrestore(&device->cache_lock, flags);
  898. return ret;
  899. }
  900. EXPORT_SYMBOL(ib_get_cached_pkey);
  901. int ib_get_cached_subnet_prefix(struct ib_device *device,
  902. u8 port_num,
  903. u64 *sn_pfx)
  904. {
  905. unsigned long flags;
  906. if (!rdma_is_port_valid(device, port_num))
  907. return -EINVAL;
  908. read_lock_irqsave(&device->cache_lock, flags);
  909. *sn_pfx = device->port_data[port_num].cache.subnet_prefix;
  910. read_unlock_irqrestore(&device->cache_lock, flags);
  911. return 0;
  912. }
  913. EXPORT_SYMBOL(ib_get_cached_subnet_prefix);
  914. int ib_find_cached_pkey(struct ib_device *device,
  915. u8 port_num,
  916. u16 pkey,
  917. u16 *index)
  918. {
  919. struct ib_pkey_cache *cache;
  920. unsigned long flags;
  921. int i;
  922. int ret = -ENOENT;
  923. int partial_ix = -1;
  924. if (!rdma_is_port_valid(device, port_num))
  925. return -EINVAL;
  926. read_lock_irqsave(&device->cache_lock, flags);
  927. cache = device->port_data[port_num].cache.pkey;
  928. if (!cache) {
  929. ret = -EINVAL;
  930. goto err;
  931. }
  932. *index = -1;
  933. for (i = 0; i < cache->table_len; ++i)
  934. if ((cache->table[i] & 0x7fff) == (pkey & 0x7fff)) {
  935. if (cache->table[i] & 0x8000) {
  936. *index = i;
  937. ret = 0;
  938. break;
  939. } else
  940. partial_ix = i;
  941. }
  942. if (ret && partial_ix >= 0) {
  943. *index = partial_ix;
  944. ret = 0;
  945. }
  946. err:
  947. read_unlock_irqrestore(&device->cache_lock, flags);
  948. return ret;
  949. }
  950. EXPORT_SYMBOL(ib_find_cached_pkey);
  951. int ib_find_exact_cached_pkey(struct ib_device *device,
  952. u8 port_num,
  953. u16 pkey,
  954. u16 *index)
  955. {
  956. struct ib_pkey_cache *cache;
  957. unsigned long flags;
  958. int i;
  959. int ret = -ENOENT;
  960. if (!rdma_is_port_valid(device, port_num))
  961. return -EINVAL;
  962. read_lock_irqsave(&device->cache_lock, flags);
  963. cache = device->port_data[port_num].cache.pkey;
  964. if (!cache) {
  965. ret = -EINVAL;
  966. goto err;
  967. }
  968. *index = -1;
  969. for (i = 0; i < cache->table_len; ++i)
  970. if (cache->table[i] == pkey) {
  971. *index = i;
  972. ret = 0;
  973. break;
  974. }
  975. err:
  976. read_unlock_irqrestore(&device->cache_lock, flags);
  977. return ret;
  978. }
  979. EXPORT_SYMBOL(ib_find_exact_cached_pkey);
  980. int ib_get_cached_lmc(struct ib_device *device,
  981. u8 port_num,
  982. u8 *lmc)
  983. {
  984. unsigned long flags;
  985. int ret = 0;
  986. if (!rdma_is_port_valid(device, port_num))
  987. return -EINVAL;
  988. read_lock_irqsave(&device->cache_lock, flags);
  989. *lmc = device->port_data[port_num].cache.lmc;
  990. read_unlock_irqrestore(&device->cache_lock, flags);
  991. return ret;
  992. }
  993. EXPORT_SYMBOL(ib_get_cached_lmc);
  994. int ib_get_cached_port_state(struct ib_device *device,
  995. u8 port_num,
  996. enum ib_port_state *port_state)
  997. {
  998. unsigned long flags;
  999. int ret = 0;
  1000. if (!rdma_is_port_valid(device, port_num))
  1001. return -EINVAL;
  1002. read_lock_irqsave(&device->cache_lock, flags);
  1003. *port_state = device->port_data[port_num].cache.port_state;
  1004. read_unlock_irqrestore(&device->cache_lock, flags);
  1005. return ret;
  1006. }
  1007. EXPORT_SYMBOL(ib_get_cached_port_state);
  1008. /**
  1009. * rdma_get_gid_attr - Returns GID attributes for a port of a device
  1010. * at a requested gid_index, if a valid GID entry exists.
  1011. * @device: The device to query.
  1012. * @port_num: The port number on the device where the GID value
  1013. * is to be queried.
  1014. * @index: Index of the GID table entry whose attributes are to
  1015. * be queried.
  1016. *
  1017. * rdma_get_gid_attr() acquires reference count of gid attributes from the
  1018. * cached GID table. Caller must invoke rdma_put_gid_attr() to release
  1019. * reference to gid attribute regardless of link layer.
  1020. *
  1021. * Returns pointer to valid gid attribute or ERR_PTR for the appropriate error
  1022. * code.
  1023. */
  1024. const struct ib_gid_attr *
  1025. rdma_get_gid_attr(struct ib_device *device, u8 port_num, int index)
  1026. {
  1027. const struct ib_gid_attr *attr = ERR_PTR(-ENODATA);
  1028. struct ib_gid_table *table;
  1029. unsigned long flags;
  1030. if (!rdma_is_port_valid(device, port_num))
  1031. return ERR_PTR(-EINVAL);
  1032. table = rdma_gid_table(device, port_num);
  1033. if (index < 0 || index >= table->sz)
  1034. return ERR_PTR(-EINVAL);
  1035. read_lock_irqsave(&table->rwlock, flags);
  1036. if (!is_gid_entry_valid(table->data_vec[index]))
  1037. goto done;
  1038. get_gid_entry(table->data_vec[index]);
  1039. attr = &table->data_vec[index]->attr;
  1040. done:
  1041. read_unlock_irqrestore(&table->rwlock, flags);
  1042. return attr;
  1043. }
  1044. EXPORT_SYMBOL(rdma_get_gid_attr);
  1045. /**
  1046. * rdma_query_gid_table - Reads GID table entries of all the ports of a device up to max_entries.
  1047. * @device: The device to query.
  1048. * @entries: Entries where GID entries are returned.
  1049. * @max_entries: Maximum number of entries that can be returned.
  1050. * Entries array must be allocated to hold max_entries number of entries.
  1051. * @num_entries: Updated to the number of entries that were successfully read.
  1052. *
  1053. * Returns number of entries on success or appropriate error code.
  1054. */
  1055. ssize_t rdma_query_gid_table(struct ib_device *device,
  1056. struct ib_uverbs_gid_entry *entries,
  1057. size_t max_entries)
  1058. {
  1059. const struct ib_gid_attr *gid_attr;
  1060. ssize_t num_entries = 0, ret;
  1061. struct ib_gid_table *table;
  1062. unsigned int port_num, i;
  1063. struct net_device *ndev;
  1064. unsigned long flags;
  1065. rdma_for_each_port(device, port_num) {
  1066. table = rdma_gid_table(device, port_num);
  1067. read_lock_irqsave(&table->rwlock, flags);
  1068. for (i = 0; i < table->sz; i++) {
  1069. if (!is_gid_entry_valid(table->data_vec[i]))
  1070. continue;
  1071. if (num_entries >= max_entries) {
  1072. ret = -EINVAL;
  1073. goto err;
  1074. }
  1075. gid_attr = &table->data_vec[i]->attr;
  1076. memcpy(&entries->gid, &gid_attr->gid,
  1077. sizeof(gid_attr->gid));
  1078. entries->gid_index = gid_attr->index;
  1079. entries->port_num = gid_attr->port_num;
  1080. entries->gid_type = gid_attr->gid_type;
  1081. ndev = rcu_dereference_protected(
  1082. gid_attr->ndev,
  1083. lockdep_is_held(&table->rwlock));
  1084. if (ndev)
  1085. entries->netdev_ifindex = ndev->ifindex;
  1086. num_entries++;
  1087. entries++;
  1088. }
  1089. read_unlock_irqrestore(&table->rwlock, flags);
  1090. }
  1091. return num_entries;
  1092. err:
  1093. read_unlock_irqrestore(&table->rwlock, flags);
  1094. return ret;
  1095. }
  1096. EXPORT_SYMBOL(rdma_query_gid_table);
  1097. /**
  1098. * rdma_put_gid_attr - Release reference to the GID attribute
  1099. * @attr: Pointer to the GID attribute whose reference
  1100. * needs to be released.
  1101. *
  1102. * rdma_put_gid_attr() must be used to release reference whose
  1103. * reference is acquired using rdma_get_gid_attr() or any APIs
  1104. * which returns a pointer to the ib_gid_attr regardless of link layer
  1105. * of IB or RoCE.
  1106. *
  1107. */
  1108. void rdma_put_gid_attr(const struct ib_gid_attr *attr)
  1109. {
  1110. struct ib_gid_table_entry *entry =
  1111. container_of(attr, struct ib_gid_table_entry, attr);
  1112. put_gid_entry(entry);
  1113. }
  1114. EXPORT_SYMBOL(rdma_put_gid_attr);
  1115. /**
  1116. * rdma_hold_gid_attr - Get reference to existing GID attribute
  1117. *
  1118. * @attr: Pointer to the GID attribute whose reference
  1119. * needs to be taken.
  1120. *
  1121. * Increase the reference count to a GID attribute to keep it from being
  1122. * freed. Callers are required to already be holding a reference to attribute.
  1123. *
  1124. */
  1125. void rdma_hold_gid_attr(const struct ib_gid_attr *attr)
  1126. {
  1127. struct ib_gid_table_entry *entry =
  1128. container_of(attr, struct ib_gid_table_entry, attr);
  1129. get_gid_entry(entry);
  1130. }
  1131. EXPORT_SYMBOL(rdma_hold_gid_attr);
  1132. /**
  1133. * rdma_read_gid_attr_ndev_rcu - Read GID attribute netdevice
  1134. * which must be in UP state.
  1135. *
  1136. * @attr:Pointer to the GID attribute
  1137. *
  1138. * Returns pointer to netdevice if the netdevice was attached to GID and
  1139. * netdevice is in UP state. Caller must hold RCU lock as this API
  1140. * reads the netdev flags which can change while netdevice migrates to
  1141. * different net namespace. Returns ERR_PTR with error code otherwise.
  1142. *
  1143. */
  1144. struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)
  1145. {
  1146. struct ib_gid_table_entry *entry =
  1147. container_of(attr, struct ib_gid_table_entry, attr);
  1148. struct ib_device *device = entry->attr.device;
  1149. struct net_device *ndev = ERR_PTR(-EINVAL);
  1150. u8 port_num = entry->attr.port_num;
  1151. struct ib_gid_table *table;
  1152. unsigned long flags;
  1153. bool valid;
  1154. table = rdma_gid_table(device, port_num);
  1155. read_lock_irqsave(&table->rwlock, flags);
  1156. valid = is_gid_entry_valid(table->data_vec[attr->index]);
  1157. if (valid) {
  1158. ndev = rcu_dereference(attr->ndev);
  1159. if (!ndev)
  1160. ndev = ERR_PTR(-ENODEV);
  1161. }
  1162. read_unlock_irqrestore(&table->rwlock, flags);
  1163. return ndev;
  1164. }
  1165. EXPORT_SYMBOL(rdma_read_gid_attr_ndev_rcu);
  1166. static int get_lower_dev_vlan(struct net_device *lower_dev,
  1167. struct netdev_nested_priv *priv)
  1168. {
  1169. u16 *vlan_id = (u16 *)priv->data;
  1170. if (is_vlan_dev(lower_dev))
  1171. *vlan_id = vlan_dev_vlan_id(lower_dev);
  1172. /* We are interested only in first level vlan device, so
  1173. * always return 1 to stop iterating over next level devices.
  1174. */
  1175. return 1;
  1176. }
  1177. /**
  1178. * rdma_read_gid_l2_fields - Read the vlan ID and source MAC address
  1179. * of a GID entry.
  1180. *
  1181. * @attr: GID attribute pointer whose L2 fields to be read
  1182. * @vlan_id: Pointer to vlan id to fill up if the GID entry has
  1183. * vlan id. It is optional.
  1184. * @smac: Pointer to smac to fill up for a GID entry. It is optional.
  1185. *
  1186. * rdma_read_gid_l2_fields() returns 0 on success and returns vlan id
  1187. * (if gid entry has vlan) and source MAC, or returns error.
  1188. */
  1189. int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,
  1190. u16 *vlan_id, u8 *smac)
  1191. {
  1192. struct netdev_nested_priv priv = {
  1193. .data = (void *)vlan_id,
  1194. };
  1195. struct net_device *ndev;
  1196. rcu_read_lock();
  1197. ndev = rcu_dereference(attr->ndev);
  1198. if (!ndev) {
  1199. rcu_read_unlock();
  1200. return -ENODEV;
  1201. }
  1202. if (smac)
  1203. ether_addr_copy(smac, ndev->dev_addr);
  1204. if (vlan_id) {
  1205. *vlan_id = 0xffff;
  1206. if (is_vlan_dev(ndev)) {
  1207. *vlan_id = vlan_dev_vlan_id(ndev);
  1208. } else {
  1209. /* If the netdev is upper device and if it's lower
  1210. * device is vlan device, consider vlan id of the
  1211. * the lower vlan device for this gid entry.
  1212. */
  1213. netdev_walk_all_lower_dev_rcu(attr->ndev,
  1214. get_lower_dev_vlan, &priv);
  1215. }
  1216. }
  1217. rcu_read_unlock();
  1218. return 0;
  1219. }
  1220. EXPORT_SYMBOL(rdma_read_gid_l2_fields);
  1221. static int config_non_roce_gid_cache(struct ib_device *device,
  1222. u8 port, int gid_tbl_len)
  1223. {
  1224. struct ib_gid_attr gid_attr = {};
  1225. struct ib_gid_table *table;
  1226. int ret = 0;
  1227. int i;
  1228. gid_attr.device = device;
  1229. gid_attr.port_num = port;
  1230. table = rdma_gid_table(device, port);
  1231. mutex_lock(&table->lock);
  1232. for (i = 0; i < gid_tbl_len; ++i) {
  1233. if (!device->ops.query_gid)
  1234. continue;
  1235. ret = device->ops.query_gid(device, port, i, &gid_attr.gid);
  1236. if (ret) {
  1237. dev_warn(&device->dev,
  1238. "query_gid failed (%d) for index %d\n", ret,
  1239. i);
  1240. goto err;
  1241. }
  1242. gid_attr.index = i;
  1243. add_modify_gid(table, &gid_attr);
  1244. }
  1245. err:
  1246. mutex_unlock(&table->lock);
  1247. return ret;
  1248. }
  1249. static int
  1250. ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
  1251. {
  1252. struct ib_port_attr *tprops = NULL;
  1253. struct ib_pkey_cache *pkey_cache = NULL, *old_pkey_cache;
  1254. int i;
  1255. int ret;
  1256. if (!rdma_is_port_valid(device, port))
  1257. return -EINVAL;
  1258. tprops = kmalloc(sizeof *tprops, GFP_KERNEL);
  1259. if (!tprops)
  1260. return -ENOMEM;
  1261. ret = ib_query_port(device, port, tprops);
  1262. if (ret) {
  1263. dev_warn(&device->dev, "ib_query_port failed (%d)\n", ret);
  1264. goto err;
  1265. }
  1266. if (!rdma_protocol_roce(device, port)) {
  1267. ret = config_non_roce_gid_cache(device, port,
  1268. tprops->gid_tbl_len);
  1269. if (ret)
  1270. goto err;
  1271. }
  1272. if (tprops->pkey_tbl_len) {
  1273. pkey_cache = kmalloc(struct_size(pkey_cache, table,
  1274. tprops->pkey_tbl_len),
  1275. GFP_KERNEL);
  1276. if (!pkey_cache) {
  1277. ret = -ENOMEM;
  1278. goto err;
  1279. }
  1280. pkey_cache->table_len = tprops->pkey_tbl_len;
  1281. for (i = 0; i < pkey_cache->table_len; ++i) {
  1282. ret = ib_query_pkey(device, port, i,
  1283. pkey_cache->table + i);
  1284. if (ret) {
  1285. dev_warn(&device->dev,
  1286. "ib_query_pkey failed (%d) for index %d\n",
  1287. ret, i);
  1288. goto err;
  1289. }
  1290. }
  1291. }
  1292. write_lock_irq(&device->cache_lock);
  1293. old_pkey_cache = device->port_data[port].cache.pkey;
  1294. device->port_data[port].cache.pkey = pkey_cache;
  1295. device->port_data[port].cache.lmc = tprops->lmc;
  1296. device->port_data[port].cache.port_state = tprops->state;
  1297. device->port_data[port].cache.subnet_prefix = tprops->subnet_prefix;
  1298. write_unlock_irq(&device->cache_lock);
  1299. if (enforce_security)
  1300. ib_security_cache_change(device,
  1301. port,
  1302. tprops->subnet_prefix);
  1303. kfree(old_pkey_cache);
  1304. kfree(tprops);
  1305. return 0;
  1306. err:
  1307. kfree(pkey_cache);
  1308. kfree(tprops);
  1309. return ret;
  1310. }
  1311. static void ib_cache_event_task(struct work_struct *_work)
  1312. {
  1313. struct ib_update_work *work =
  1314. container_of(_work, struct ib_update_work, work);
  1315. int ret;
  1316. /* Before distributing the cache update event, first sync
  1317. * the cache.
  1318. */
  1319. ret = ib_cache_update(work->event.device, work->event.element.port_num,
  1320. work->enforce_security);
  1321. /* GID event is notified already for individual GID entries by
  1322. * dispatch_gid_change_event(). Hence, notifiy for rest of the
  1323. * events.
  1324. */
  1325. if (!ret && work->event.event != IB_EVENT_GID_CHANGE)
  1326. ib_dispatch_event_clients(&work->event);
  1327. kfree(work);
  1328. }
  1329. static void ib_generic_event_task(struct work_struct *_work)
  1330. {
  1331. struct ib_update_work *work =
  1332. container_of(_work, struct ib_update_work, work);
  1333. ib_dispatch_event_clients(&work->event);
  1334. kfree(work);
  1335. }
  1336. static bool is_cache_update_event(const struct ib_event *event)
  1337. {
  1338. return (event->event == IB_EVENT_PORT_ERR ||
  1339. event->event == IB_EVENT_PORT_ACTIVE ||
  1340. event->event == IB_EVENT_LID_CHANGE ||
  1341. event->event == IB_EVENT_PKEY_CHANGE ||
  1342. event->event == IB_EVENT_CLIENT_REREGISTER ||
  1343. event->event == IB_EVENT_GID_CHANGE);
  1344. }
  1345. /**
  1346. * ib_dispatch_event - Dispatch an asynchronous event
  1347. * @event:Event to dispatch
  1348. *
  1349. * Low-level drivers must call ib_dispatch_event() to dispatch the
  1350. * event to all registered event handlers when an asynchronous event
  1351. * occurs.
  1352. */
  1353. void ib_dispatch_event(const struct ib_event *event)
  1354. {
  1355. struct ib_update_work *work;
  1356. work = kzalloc(sizeof(*work), GFP_ATOMIC);
  1357. if (!work)
  1358. return;
  1359. if (is_cache_update_event(event))
  1360. INIT_WORK(&work->work, ib_cache_event_task);
  1361. else
  1362. INIT_WORK(&work->work, ib_generic_event_task);
  1363. work->event = *event;
  1364. if (event->event == IB_EVENT_PKEY_CHANGE ||
  1365. event->event == IB_EVENT_GID_CHANGE)
  1366. work->enforce_security = true;
  1367. queue_work(ib_wq, &work->work);
  1368. }
  1369. EXPORT_SYMBOL(ib_dispatch_event);
  1370. int ib_cache_setup_one(struct ib_device *device)
  1371. {
  1372. unsigned int p;
  1373. int err;
  1374. rwlock_init(&device->cache_lock);
  1375. err = gid_table_setup_one(device);
  1376. if (err)
  1377. return err;
  1378. rdma_for_each_port (device, p) {
  1379. err = ib_cache_update(device, p, true);
  1380. if (err)
  1381. return err;
  1382. }
  1383. return 0;
  1384. }
  1385. void ib_cache_release_one(struct ib_device *device)
  1386. {
  1387. unsigned int p;
  1388. /*
  1389. * The release function frees all the cache elements.
  1390. * This function should be called as part of freeing
  1391. * all the device's resources when the cache could no
  1392. * longer be accessed.
  1393. */
  1394. rdma_for_each_port (device, p)
  1395. kfree(device->port_data[p].cache.pkey);
  1396. gid_table_release_one(device);
  1397. }
  1398. void ib_cache_cleanup_one(struct ib_device *device)
  1399. {
  1400. /* The cleanup function waits for all in-progress workqueue
  1401. * elements and cleans up the GID cache. This function should be
  1402. * called after the device was removed from the devices list and
  1403. * all clients were removed, so the cache exists but is
  1404. * non-functional and shouldn't be updated anymore.
  1405. */
  1406. flush_workqueue(ib_wq);
  1407. gid_table_cleanup_one(device);
  1408. /*
  1409. * Flush the wq second time for any pending GID delete work.
  1410. */
  1411. flush_workqueue(ib_wq);
  1412. }