xhci-ring.c 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * USB HOST XHCI Controller stack
  4. *
  5. * Based on xHCI host controller driver in linux-kernel
  6. * by Sarah Sharp.
  7. *
  8. * Copyright (C) 2008 Intel Corp.
  9. * Author: Sarah Sharp
  10. *
  11. * Copyright (C) 2013 Samsung Electronics Co.Ltd
  12. * Authors: Vivek Gautam <gautam.vivek@samsung.com>
  13. * Vikas Sajjan <vikas.sajjan@samsung.com>
  14. */
  15. #include <common.h>
  16. #include <cpu_func.h>
  17. #include <log.h>
  18. #include <asm/byteorder.h>
  19. #include <usb.h>
  20. #include <asm/unaligned.h>
  21. #include <linux/bug.h>
  22. #include <linux/errno.h>
  23. #include <usb/xhci.h>
  24. /**
  25. * Is this TRB a link TRB or was the last TRB the last TRB in this event ring
  26. * segment? I.e. would the updated event TRB pointer step off the end of the
  27. * event seg ?
  28. *
  29. * @param ctrl Host controller data structure
  30. * @param ring pointer to the ring
  31. * @param seg poniter to the segment to which TRB belongs
  32. * @param trb poniter to the ring trb
  33. * @return 1 if this TRB a link TRB else 0
  34. */
  35. static int last_trb(struct xhci_ctrl *ctrl, struct xhci_ring *ring,
  36. struct xhci_segment *seg, union xhci_trb *trb)
  37. {
  38. if (ring == ctrl->event_ring)
  39. return trb == &seg->trbs[TRBS_PER_SEGMENT];
  40. else
  41. return TRB_TYPE_LINK_LE32(trb->link.control);
  42. }
  43. /**
  44. * Does this link TRB point to the first segment in a ring,
  45. * or was the previous TRB the last TRB on the last segment in the ERST?
  46. *
  47. * @param ctrl Host controller data structure
  48. * @param ring pointer to the ring
  49. * @param seg poniter to the segment to which TRB belongs
  50. * @param trb poniter to the ring trb
  51. * @return 1 if this TRB is the last TRB on the last segment else 0
  52. */
  53. static bool last_trb_on_last_seg(struct xhci_ctrl *ctrl,
  54. struct xhci_ring *ring,
  55. struct xhci_segment *seg,
  56. union xhci_trb *trb)
  57. {
  58. if (ring == ctrl->event_ring)
  59. return ((trb == &seg->trbs[TRBS_PER_SEGMENT]) &&
  60. (seg->next == ring->first_seg));
  61. else
  62. return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
  63. }
  64. /**
  65. * See Cycle bit rules. SW is the consumer for the event ring only.
  66. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  67. *
  68. * If we've just enqueued a TRB that is in the middle of a TD (meaning the
  69. * chain bit is set), then set the chain bit in all the following link TRBs.
  70. * If we've enqueued the last TRB in a TD, make sure the following link TRBs
  71. * have their chain bit cleared (so that each Link TRB is a separate TD).
  72. *
  73. * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
  74. * set, but other sections talk about dealing with the chain bit set. This was
  75. * fixed in the 0.96 specification errata, but we have to assume that all 0.95
  76. * xHCI hardware can't handle the chain bit being cleared on a link TRB.
  77. *
  78. * @param ctrl Host controller data structure
  79. * @param ring pointer to the ring
  80. * @param more_trbs_coming flag to indicate whether more trbs
  81. * are expected or NOT.
  82. * Will you enqueue more TRBs before calling
  83. * prepare_ring()?
  84. * @return none
  85. */
  86. static void inc_enq(struct xhci_ctrl *ctrl, struct xhci_ring *ring,
  87. bool more_trbs_coming)
  88. {
  89. u32 chain;
  90. union xhci_trb *next;
  91. chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
  92. next = ++(ring->enqueue);
  93. /*
  94. * Update the dequeue pointer further if that was a link TRB or we're at
  95. * the end of an event ring segment (which doesn't have link TRBS)
  96. */
  97. while (last_trb(ctrl, ring, ring->enq_seg, next)) {
  98. if (ring != ctrl->event_ring) {
  99. /*
  100. * If the caller doesn't plan on enqueueing more
  101. * TDs before ringing the doorbell, then we
  102. * don't want to give the link TRB to the
  103. * hardware just yet. We'll give the link TRB
  104. * back in prepare_ring() just before we enqueue
  105. * the TD at the top of the ring.
  106. */
  107. if (!chain && !more_trbs_coming)
  108. break;
  109. /*
  110. * If we're not dealing with 0.95 hardware or
  111. * isoc rings on AMD 0.96 host,
  112. * carry over the chain bit of the previous TRB
  113. * (which may mean the chain bit is cleared).
  114. */
  115. next->link.control &= cpu_to_le32(~TRB_CHAIN);
  116. next->link.control |= cpu_to_le32(chain);
  117. next->link.control ^= cpu_to_le32(TRB_CYCLE);
  118. xhci_flush_cache((uintptr_t)next,
  119. sizeof(union xhci_trb));
  120. }
  121. /* Toggle the cycle bit after the last ring segment. */
  122. if (last_trb_on_last_seg(ctrl, ring,
  123. ring->enq_seg, next))
  124. ring->cycle_state = (ring->cycle_state ? 0 : 1);
  125. ring->enq_seg = ring->enq_seg->next;
  126. ring->enqueue = ring->enq_seg->trbs;
  127. next = ring->enqueue;
  128. }
  129. }
  130. /**
  131. * See Cycle bit rules. SW is the consumer for the event ring only.
  132. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  133. *
  134. * @param ctrl Host controller data structure
  135. * @param ring Ring whose Dequeue TRB pointer needs to be incremented.
  136. * return none
  137. */
  138. static void inc_deq(struct xhci_ctrl *ctrl, struct xhci_ring *ring)
  139. {
  140. do {
  141. /*
  142. * Update the dequeue pointer further if that was a link TRB or
  143. * we're at the end of an event ring segment (which doesn't have
  144. * link TRBS)
  145. */
  146. if (last_trb(ctrl, ring, ring->deq_seg, ring->dequeue)) {
  147. if (ring == ctrl->event_ring &&
  148. last_trb_on_last_seg(ctrl, ring,
  149. ring->deq_seg, ring->dequeue)) {
  150. ring->cycle_state = (ring->cycle_state ? 0 : 1);
  151. }
  152. ring->deq_seg = ring->deq_seg->next;
  153. ring->dequeue = ring->deq_seg->trbs;
  154. } else {
  155. ring->dequeue++;
  156. }
  157. } while (last_trb(ctrl, ring, ring->deq_seg, ring->dequeue));
  158. }
  159. /**
  160. * Generic function for queueing a TRB on a ring.
  161. * The caller must have checked to make sure there's room on the ring.
  162. *
  163. * @param more_trbs_coming: Will you enqueue more TRBs before calling
  164. * prepare_ring()?
  165. * @param ctrl Host controller data structure
  166. * @param ring pointer to the ring
  167. * @param more_trbs_coming flag to indicate whether more trbs
  168. * @param trb_fields pointer to trb field array containing TRB contents
  169. * @return pointer to the enqueued trb
  170. */
  171. static struct xhci_generic_trb *queue_trb(struct xhci_ctrl *ctrl,
  172. struct xhci_ring *ring,
  173. bool more_trbs_coming,
  174. unsigned int *trb_fields)
  175. {
  176. struct xhci_generic_trb *trb;
  177. int i;
  178. trb = &ring->enqueue->generic;
  179. for (i = 0; i < 4; i++)
  180. trb->field[i] = cpu_to_le32(trb_fields[i]);
  181. xhci_flush_cache((uintptr_t)trb, sizeof(struct xhci_generic_trb));
  182. inc_enq(ctrl, ring, more_trbs_coming);
  183. return trb;
  184. }
  185. /**
  186. * Does various checks on the endpoint ring, and makes it ready
  187. * to queue num_trbs.
  188. *
  189. * @param ctrl Host controller data structure
  190. * @param ep_ring pointer to the EP Transfer Ring
  191. * @param ep_state State of the End Point
  192. * @return error code in case of invalid ep_state, 0 on success
  193. */
  194. static int prepare_ring(struct xhci_ctrl *ctrl, struct xhci_ring *ep_ring,
  195. u32 ep_state)
  196. {
  197. union xhci_trb *next = ep_ring->enqueue;
  198. /* Make sure the endpoint has been added to xHC schedule */
  199. switch (ep_state) {
  200. case EP_STATE_DISABLED:
  201. /*
  202. * USB core changed config/interfaces without notifying us,
  203. * or hardware is reporting the wrong state.
  204. */
  205. puts("WARN urb submitted to disabled ep\n");
  206. return -ENOENT;
  207. case EP_STATE_ERROR:
  208. puts("WARN waiting for error on ep to be cleared\n");
  209. return -EINVAL;
  210. case EP_STATE_HALTED:
  211. puts("WARN halted endpoint, queueing URB anyway.\n");
  212. case EP_STATE_STOPPED:
  213. case EP_STATE_RUNNING:
  214. debug("EP STATE RUNNING.\n");
  215. break;
  216. default:
  217. puts("ERROR unknown endpoint state for ep\n");
  218. return -EINVAL;
  219. }
  220. while (last_trb(ctrl, ep_ring, ep_ring->enq_seg, next)) {
  221. /*
  222. * If we're not dealing with 0.95 hardware or isoc rings
  223. * on AMD 0.96 host, clear the chain bit.
  224. */
  225. next->link.control &= cpu_to_le32(~TRB_CHAIN);
  226. next->link.control ^= cpu_to_le32(TRB_CYCLE);
  227. xhci_flush_cache((uintptr_t)next, sizeof(union xhci_trb));
  228. /* Toggle the cycle bit after the last ring segment. */
  229. if (last_trb_on_last_seg(ctrl, ep_ring,
  230. ep_ring->enq_seg, next))
  231. ep_ring->cycle_state = (ep_ring->cycle_state ? 0 : 1);
  232. ep_ring->enq_seg = ep_ring->enq_seg->next;
  233. ep_ring->enqueue = ep_ring->enq_seg->trbs;
  234. next = ep_ring->enqueue;
  235. }
  236. return 0;
  237. }
  238. /**
  239. * Generic function for queueing a command TRB on the command ring.
  240. * Check to make sure there's room on the command ring for one command TRB.
  241. *
  242. * @param ctrl Host controller data structure
  243. * @param ptr Pointer address to write in the first two fields (opt.)
  244. * @param slot_id Slot ID to encode in the flags field (opt.)
  245. * @param ep_index Endpoint index to encode in the flags field (opt.)
  246. * @param cmd Command type to enqueue
  247. * @return none
  248. */
  249. void xhci_queue_command(struct xhci_ctrl *ctrl, u8 *ptr, u32 slot_id,
  250. u32 ep_index, trb_type cmd)
  251. {
  252. u32 fields[4];
  253. u64 val_64 = virt_to_phys(ptr);
  254. BUG_ON(prepare_ring(ctrl, ctrl->cmd_ring, EP_STATE_RUNNING));
  255. fields[0] = lower_32_bits(val_64);
  256. fields[1] = upper_32_bits(val_64);
  257. fields[2] = 0;
  258. fields[3] = TRB_TYPE(cmd) | SLOT_ID_FOR_TRB(slot_id) |
  259. ctrl->cmd_ring->cycle_state;
  260. /*
  261. * Only 'reset endpoint', 'stop endpoint' and 'set TR dequeue pointer'
  262. * commands need endpoint id encoded.
  263. */
  264. if (cmd >= TRB_RESET_EP && cmd <= TRB_SET_DEQ)
  265. fields[3] |= EP_ID_FOR_TRB(ep_index);
  266. queue_trb(ctrl, ctrl->cmd_ring, false, fields);
  267. /* Ring the command ring doorbell */
  268. xhci_writel(&ctrl->dba->doorbell[0], DB_VALUE_HOST);
  269. }
  270. /*
  271. * For xHCI 1.0 host controllers, TD size is the number of max packet sized
  272. * packets remaining in the TD (*not* including this TRB).
  273. *
  274. * Total TD packet count = total_packet_count =
  275. * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
  276. *
  277. * Packets transferred up to and including this TRB = packets_transferred =
  278. * rounddown(total bytes transferred including this TRB / wMaxPacketSize)
  279. *
  280. * TD size = total_packet_count - packets_transferred
  281. *
  282. * For xHCI 0.96 and older, TD size field should be the remaining bytes
  283. * including this TRB, right shifted by 10
  284. *
  285. * For all hosts it must fit in bits 21:17, so it can't be bigger than 31.
  286. * This is taken care of in the TRB_TD_SIZE() macro
  287. *
  288. * The last TRB in a TD must have the TD size set to zero.
  289. *
  290. * @param ctrl host controller data structure
  291. * @param transferred total size sent so far
  292. * @param trb_buff_len length of the TRB Buffer
  293. * @param td_total_len total packet count
  294. * @param maxp max packet size of current pipe
  295. * @param more_trbs_coming indicate last trb in TD
  296. * @return remainder
  297. */
  298. static u32 xhci_td_remainder(struct xhci_ctrl *ctrl, int transferred,
  299. int trb_buff_len, unsigned int td_total_len,
  300. int maxp, bool more_trbs_coming)
  301. {
  302. u32 total_packet_count;
  303. /* MTK xHCI 0.96 contains some features from 1.0 */
  304. if (ctrl->hci_version < 0x100 && !(ctrl->quirks & XHCI_MTK_HOST))
  305. return ((td_total_len - transferred) >> 10);
  306. /* One TRB with a zero-length data packet. */
  307. if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
  308. trb_buff_len == td_total_len)
  309. return 0;
  310. /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */
  311. if ((ctrl->quirks & XHCI_MTK_HOST) && (ctrl->hci_version < 0x100))
  312. trb_buff_len = 0;
  313. total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
  314. /* Queueing functions don't count the current TRB into transferred */
  315. return (total_packet_count - ((transferred + trb_buff_len) / maxp));
  316. }
  317. /**
  318. * Ring the doorbell of the End Point
  319. *
  320. * @param udev pointer to the USB device structure
  321. * @param ep_index index of the endpoint
  322. * @param start_cycle cycle flag of the first TRB
  323. * @param start_trb pionter to the first TRB
  324. * @return none
  325. */
  326. static void giveback_first_trb(struct usb_device *udev, int ep_index,
  327. int start_cycle,
  328. struct xhci_generic_trb *start_trb)
  329. {
  330. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  331. /*
  332. * Pass all the TRBs to the hardware at once and make sure this write
  333. * isn't reordered.
  334. */
  335. if (start_cycle)
  336. start_trb->field[3] |= cpu_to_le32(start_cycle);
  337. else
  338. start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
  339. xhci_flush_cache((uintptr_t)start_trb, sizeof(struct xhci_generic_trb));
  340. /* Ringing EP doorbell here */
  341. xhci_writel(&ctrl->dba->doorbell[udev->slot_id],
  342. DB_VALUE(ep_index, 0));
  343. return;
  344. }
  345. /**** POLLING mechanism for XHCI ****/
  346. /**
  347. * Finalizes a handled event TRB by advancing our dequeue pointer and giving
  348. * the TRB back to the hardware for recycling. Must call this exactly once at
  349. * the end of each event handler, and not touch the TRB again afterwards.
  350. *
  351. * @param ctrl Host controller data structure
  352. * @return none
  353. */
  354. void xhci_acknowledge_event(struct xhci_ctrl *ctrl)
  355. {
  356. /* Advance our dequeue pointer to the next event */
  357. inc_deq(ctrl, ctrl->event_ring);
  358. /* Inform the hardware */
  359. xhci_writeq(&ctrl->ir_set->erst_dequeue,
  360. virt_to_phys(ctrl->event_ring->dequeue) | ERST_EHB);
  361. }
  362. /**
  363. * Checks if there is a new event to handle on the event ring.
  364. *
  365. * @param ctrl Host controller data structure
  366. * @return 0 if failure else 1 on success
  367. */
  368. static int event_ready(struct xhci_ctrl *ctrl)
  369. {
  370. union xhci_trb *event;
  371. xhci_inval_cache((uintptr_t)ctrl->event_ring->dequeue,
  372. sizeof(union xhci_trb));
  373. event = ctrl->event_ring->dequeue;
  374. /* Does the HC or OS own the TRB? */
  375. if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
  376. ctrl->event_ring->cycle_state)
  377. return 0;
  378. return 1;
  379. }
  380. /**
  381. * Waits for a specific type of event and returns it. Discards unexpected
  382. * events. Caller *must* call xhci_acknowledge_event() after it is finished
  383. * processing the event, and must not access the returned pointer afterwards.
  384. *
  385. * @param ctrl Host controller data structure
  386. * @param expected TRB type expected from Event TRB
  387. * @return pointer to event trb
  388. */
  389. union xhci_trb *xhci_wait_for_event(struct xhci_ctrl *ctrl, trb_type expected)
  390. {
  391. trb_type type;
  392. unsigned long ts = get_timer(0);
  393. do {
  394. union xhci_trb *event = ctrl->event_ring->dequeue;
  395. if (!event_ready(ctrl))
  396. continue;
  397. type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags));
  398. if (type == expected)
  399. return event;
  400. if (type == TRB_PORT_STATUS)
  401. /* TODO: remove this once enumeration has been reworked */
  402. /*
  403. * Port status change events always have a
  404. * successful completion code
  405. */
  406. BUG_ON(GET_COMP_CODE(
  407. le32_to_cpu(event->generic.field[2])) !=
  408. COMP_SUCCESS);
  409. else
  410. printf("Unexpected XHCI event TRB, skipping... "
  411. "(%08x %08x %08x %08x)\n",
  412. le32_to_cpu(event->generic.field[0]),
  413. le32_to_cpu(event->generic.field[1]),
  414. le32_to_cpu(event->generic.field[2]),
  415. le32_to_cpu(event->generic.field[3]));
  416. xhci_acknowledge_event(ctrl);
  417. } while (get_timer(ts) < XHCI_TIMEOUT);
  418. if (expected == TRB_TRANSFER)
  419. return NULL;
  420. printf("XHCI timeout on event type %d... cannot recover.\n", expected);
  421. BUG();
  422. }
  423. /*
  424. * Stops transfer processing for an endpoint and throws away all unprocessed
  425. * TRBs by setting the xHC's dequeue pointer to our enqueue pointer. The next
  426. * xhci_bulk_tx/xhci_ctrl_tx on this enpoint will add new transfers there and
  427. * ring the doorbell, causing this endpoint to start working again.
  428. * (Careful: This will BUG() when there was no transfer in progress. Shouldn't
  429. * happen in practice for current uses and is too complicated to fix right now.)
  430. */
  431. static void abort_td(struct usb_device *udev, int ep_index)
  432. {
  433. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  434. struct xhci_ring *ring = ctrl->devs[udev->slot_id]->eps[ep_index].ring;
  435. union xhci_trb *event;
  436. u32 field;
  437. xhci_queue_command(ctrl, NULL, udev->slot_id, ep_index, TRB_STOP_RING);
  438. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  439. field = le32_to_cpu(event->trans_event.flags);
  440. BUG_ON(TRB_TO_SLOT_ID(field) != udev->slot_id);
  441. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  442. BUG_ON(GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len
  443. != COMP_STOP)));
  444. xhci_acknowledge_event(ctrl);
  445. event = xhci_wait_for_event(ctrl, TRB_COMPLETION);
  446. BUG_ON(TRB_TO_SLOT_ID(le32_to_cpu(event->event_cmd.flags))
  447. != udev->slot_id || GET_COMP_CODE(le32_to_cpu(
  448. event->event_cmd.status)) != COMP_SUCCESS);
  449. xhci_acknowledge_event(ctrl);
  450. xhci_queue_command(ctrl, (void *)((uintptr_t)ring->enqueue |
  451. ring->cycle_state), udev->slot_id, ep_index, TRB_SET_DEQ);
  452. event = xhci_wait_for_event(ctrl, TRB_COMPLETION);
  453. BUG_ON(TRB_TO_SLOT_ID(le32_to_cpu(event->event_cmd.flags))
  454. != udev->slot_id || GET_COMP_CODE(le32_to_cpu(
  455. event->event_cmd.status)) != COMP_SUCCESS);
  456. xhci_acknowledge_event(ctrl);
  457. }
  458. static void record_transfer_result(struct usb_device *udev,
  459. union xhci_trb *event, int length)
  460. {
  461. udev->act_len = min(length, length -
  462. (int)EVENT_TRB_LEN(le32_to_cpu(event->trans_event.transfer_len)));
  463. switch (GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len))) {
  464. case COMP_SUCCESS:
  465. BUG_ON(udev->act_len != length);
  466. /* fallthrough */
  467. case COMP_SHORT_TX:
  468. udev->status = 0;
  469. break;
  470. case COMP_STALL:
  471. udev->status = USB_ST_STALLED;
  472. break;
  473. case COMP_DB_ERR:
  474. case COMP_TRB_ERR:
  475. udev->status = USB_ST_BUF_ERR;
  476. break;
  477. case COMP_BABBLE:
  478. udev->status = USB_ST_BABBLE_DET;
  479. break;
  480. default:
  481. udev->status = 0x80; /* USB_ST_TOO_LAZY_TO_MAKE_A_NEW_MACRO */
  482. }
  483. }
  484. /**** Bulk and Control transfer methods ****/
  485. /**
  486. * Queues up the BULK Request
  487. *
  488. * @param udev pointer to the USB device structure
  489. * @param pipe contains the DIR_IN or OUT , devnum
  490. * @param length length of the buffer
  491. * @param buffer buffer to be read/written based on the request
  492. * @return returns 0 if successful else -1 on failure
  493. */
  494. int xhci_bulk_tx(struct usb_device *udev, unsigned long pipe,
  495. int length, void *buffer)
  496. {
  497. int num_trbs = 0;
  498. struct xhci_generic_trb *start_trb;
  499. bool first_trb = false;
  500. int start_cycle;
  501. u32 field = 0;
  502. u32 length_field = 0;
  503. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  504. int slot_id = udev->slot_id;
  505. int ep_index;
  506. struct xhci_virt_device *virt_dev;
  507. struct xhci_ep_ctx *ep_ctx;
  508. struct xhci_ring *ring; /* EP transfer ring */
  509. union xhci_trb *event;
  510. int running_total, trb_buff_len;
  511. bool more_trbs_coming = true;
  512. int maxpacketsize;
  513. u64 addr;
  514. int ret;
  515. u32 trb_fields[4];
  516. u64 val_64 = virt_to_phys(buffer);
  517. debug("dev=%p, pipe=%lx, buffer=%p, length=%d\n",
  518. udev, pipe, buffer, length);
  519. ep_index = usb_pipe_ep_index(pipe);
  520. virt_dev = ctrl->devs[slot_id];
  521. xhci_inval_cache((uintptr_t)virt_dev->out_ctx->bytes,
  522. virt_dev->out_ctx->size);
  523. ep_ctx = xhci_get_ep_ctx(ctrl, virt_dev->out_ctx, ep_index);
  524. ring = virt_dev->eps[ep_index].ring;
  525. /*
  526. * How much data is (potentially) left before the 64KB boundary?
  527. * XHCI Spec puts restriction( TABLE 49 and 6.4.1 section of XHCI Spec)
  528. * that the buffer should not span 64KB boundary. if so
  529. * we send request in more than 1 TRB by chaining them.
  530. */
  531. running_total = TRB_MAX_BUFF_SIZE -
  532. (lower_32_bits(val_64) & (TRB_MAX_BUFF_SIZE - 1));
  533. trb_buff_len = running_total;
  534. running_total &= TRB_MAX_BUFF_SIZE - 1;
  535. /*
  536. * If there's some data on this 64KB chunk, or we have to send a
  537. * zero-length transfer, we need at least one TRB
  538. */
  539. if (running_total != 0 || length == 0)
  540. num_trbs++;
  541. /* How many more 64KB chunks to transfer, how many more TRBs? */
  542. while (running_total < length) {
  543. num_trbs++;
  544. running_total += TRB_MAX_BUFF_SIZE;
  545. }
  546. /*
  547. * XXX: Calling routine prepare_ring() called in place of
  548. * prepare_trasfer() as there in 'Linux' since we are not
  549. * maintaining multiple TDs/transfer at the same time.
  550. */
  551. ret = prepare_ring(ctrl, ring,
  552. le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK);
  553. if (ret < 0)
  554. return ret;
  555. /*
  556. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  557. * until we've finished creating all the other TRBs. The ring's cycle
  558. * state may change as we enqueue the other TRBs, so save it too.
  559. */
  560. start_trb = &ring->enqueue->generic;
  561. start_cycle = ring->cycle_state;
  562. running_total = 0;
  563. maxpacketsize = usb_maxpacket(udev, pipe);
  564. /* How much data is in the first TRB? */
  565. /*
  566. * How much data is (potentially) left before the 64KB boundary?
  567. * XHCI Spec puts restriction( TABLE 49 and 6.4.1 section of XHCI Spec)
  568. * that the buffer should not span 64KB boundary. if so
  569. * we send request in more than 1 TRB by chaining them.
  570. */
  571. addr = val_64;
  572. if (trb_buff_len > length)
  573. trb_buff_len = length;
  574. first_trb = true;
  575. /* flush the buffer before use */
  576. xhci_flush_cache((uintptr_t)buffer, length);
  577. /* Queue the first TRB, even if it's zero-length */
  578. do {
  579. u32 remainder = 0;
  580. field = 0;
  581. /* Don't change the cycle bit of the first TRB until later */
  582. if (first_trb) {
  583. first_trb = false;
  584. if (start_cycle == 0)
  585. field |= TRB_CYCLE;
  586. } else {
  587. field |= ring->cycle_state;
  588. }
  589. /*
  590. * Chain all the TRBs together; clear the chain bit in the last
  591. * TRB to indicate it's the last TRB in the chain.
  592. */
  593. if (num_trbs > 1) {
  594. field |= TRB_CHAIN;
  595. } else {
  596. field |= TRB_IOC;
  597. more_trbs_coming = false;
  598. }
  599. /* Only set interrupt on short packet for IN endpoints */
  600. if (usb_pipein(pipe))
  601. field |= TRB_ISP;
  602. /* Set the TRB length, TD size, and interrupter fields. */
  603. remainder = xhci_td_remainder(ctrl, running_total, trb_buff_len,
  604. length, maxpacketsize,
  605. more_trbs_coming);
  606. length_field = (TRB_LEN(trb_buff_len) |
  607. TRB_TD_SIZE(remainder) |
  608. TRB_INTR_TARGET(0));
  609. trb_fields[0] = lower_32_bits(addr);
  610. trb_fields[1] = upper_32_bits(addr);
  611. trb_fields[2] = length_field;
  612. trb_fields[3] = field | TRB_TYPE(TRB_NORMAL);
  613. queue_trb(ctrl, ring, (num_trbs > 1), trb_fields);
  614. --num_trbs;
  615. running_total += trb_buff_len;
  616. /* Calculate length for next transfer */
  617. addr += trb_buff_len;
  618. trb_buff_len = min((length - running_total), TRB_MAX_BUFF_SIZE);
  619. } while (running_total < length);
  620. giveback_first_trb(udev, ep_index, start_cycle, start_trb);
  621. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  622. if (!event) {
  623. debug("XHCI bulk transfer timed out, aborting...\n");
  624. abort_td(udev, ep_index);
  625. udev->status = USB_ST_NAK_REC; /* closest thing to a timeout */
  626. udev->act_len = 0;
  627. return -ETIMEDOUT;
  628. }
  629. field = le32_to_cpu(event->trans_event.flags);
  630. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  631. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  632. BUG_ON(*(void **)(uintptr_t)le64_to_cpu(event->trans_event.buffer) -
  633. buffer > (size_t)length);
  634. record_transfer_result(udev, event, length);
  635. xhci_acknowledge_event(ctrl);
  636. xhci_inval_cache((uintptr_t)buffer, length);
  637. return (udev->status != USB_ST_NOT_PROC) ? 0 : -1;
  638. }
  639. /**
  640. * Queues up the Control Transfer Request
  641. *
  642. * @param udev pointer to the USB device structure
  643. * @param pipe contains the DIR_IN or OUT , devnum
  644. * @param req request type
  645. * @param length length of the buffer
  646. * @param buffer buffer to be read/written based on the request
  647. * @return returns 0 if successful else error code on failure
  648. */
  649. int xhci_ctrl_tx(struct usb_device *udev, unsigned long pipe,
  650. struct devrequest *req, int length,
  651. void *buffer)
  652. {
  653. int ret;
  654. int start_cycle;
  655. int num_trbs;
  656. u32 field;
  657. u32 length_field;
  658. u64 buf_64 = 0;
  659. struct xhci_generic_trb *start_trb;
  660. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  661. int slot_id = udev->slot_id;
  662. int ep_index;
  663. u32 trb_fields[4];
  664. struct xhci_virt_device *virt_dev = ctrl->devs[slot_id];
  665. struct xhci_ring *ep_ring;
  666. union xhci_trb *event;
  667. u32 remainder;
  668. debug("req=%u (%#x), type=%u (%#x), value=%u (%#x), index=%u\n",
  669. req->request, req->request,
  670. req->requesttype, req->requesttype,
  671. le16_to_cpu(req->value), le16_to_cpu(req->value),
  672. le16_to_cpu(req->index));
  673. ep_index = usb_pipe_ep_index(pipe);
  674. ep_ring = virt_dev->eps[ep_index].ring;
  675. /*
  676. * Check to see if the max packet size for the default control
  677. * endpoint changed during FS device enumeration
  678. */
  679. if (udev->speed == USB_SPEED_FULL) {
  680. ret = xhci_check_maxpacket(udev);
  681. if (ret < 0)
  682. return ret;
  683. }
  684. xhci_inval_cache((uintptr_t)virt_dev->out_ctx->bytes,
  685. virt_dev->out_ctx->size);
  686. struct xhci_ep_ctx *ep_ctx = NULL;
  687. ep_ctx = xhci_get_ep_ctx(ctrl, virt_dev->out_ctx, ep_index);
  688. /* 1 TRB for setup, 1 for status */
  689. num_trbs = 2;
  690. /*
  691. * Don't need to check if we need additional event data and normal TRBs,
  692. * since data in control transfers will never get bigger than 16MB
  693. * XXX: can we get a buffer that crosses 64KB boundaries?
  694. */
  695. if (length > 0)
  696. num_trbs++;
  697. /*
  698. * XXX: Calling routine prepare_ring() called in place of
  699. * prepare_trasfer() as there in 'Linux' since we are not
  700. * maintaining multiple TDs/transfer at the same time.
  701. */
  702. ret = prepare_ring(ctrl, ep_ring,
  703. le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK);
  704. if (ret < 0)
  705. return ret;
  706. /*
  707. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  708. * until we've finished creating all the other TRBs. The ring's cycle
  709. * state may change as we enqueue the other TRBs, so save it too.
  710. */
  711. start_trb = &ep_ring->enqueue->generic;
  712. start_cycle = ep_ring->cycle_state;
  713. debug("start_trb %p, start_cycle %d\n", start_trb, start_cycle);
  714. /* Queue setup TRB - see section 6.4.1.2.1 */
  715. /* FIXME better way to translate setup_packet into two u32 fields? */
  716. field = 0;
  717. field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
  718. if (start_cycle == 0)
  719. field |= 0x1;
  720. /* xHCI 1.0 6.4.1.2.1: Transfer Type field */
  721. if (ctrl->hci_version >= 0x100 || ctrl->quirks & XHCI_MTK_HOST) {
  722. if (length > 0) {
  723. if (req->requesttype & USB_DIR_IN)
  724. field |= TRB_TX_TYPE(TRB_DATA_IN);
  725. else
  726. field |= TRB_TX_TYPE(TRB_DATA_OUT);
  727. }
  728. }
  729. debug("req->requesttype = %d, req->request = %d,"
  730. "le16_to_cpu(req->value) = %d,"
  731. "le16_to_cpu(req->index) = %d,"
  732. "le16_to_cpu(req->length) = %d\n",
  733. req->requesttype, req->request, le16_to_cpu(req->value),
  734. le16_to_cpu(req->index), le16_to_cpu(req->length));
  735. trb_fields[0] = req->requesttype | req->request << 8 |
  736. le16_to_cpu(req->value) << 16;
  737. trb_fields[1] = le16_to_cpu(req->index) |
  738. le16_to_cpu(req->length) << 16;
  739. /* TRB_LEN | (TRB_INTR_TARGET) */
  740. trb_fields[2] = (TRB_LEN(8) | TRB_INTR_TARGET(0));
  741. /* Immediate data in pointer */
  742. trb_fields[3] = field;
  743. queue_trb(ctrl, ep_ring, true, trb_fields);
  744. /* Re-initializing field to zero */
  745. field = 0;
  746. /* If there's data, queue data TRBs */
  747. /* Only set interrupt on short packet for IN endpoints */
  748. if (usb_pipein(pipe))
  749. field = TRB_ISP | TRB_TYPE(TRB_DATA);
  750. else
  751. field = TRB_TYPE(TRB_DATA);
  752. remainder = xhci_td_remainder(ctrl, 0, length, length,
  753. usb_maxpacket(udev, pipe), true);
  754. length_field = TRB_LEN(length) | TRB_TD_SIZE(remainder) |
  755. TRB_INTR_TARGET(0);
  756. debug("length_field = %d, length = %d,"
  757. "xhci_td_remainder(length) = %d , TRB_INTR_TARGET(0) = %d\n",
  758. length_field, TRB_LEN(length),
  759. TRB_TD_SIZE(remainder), 0);
  760. if (length > 0) {
  761. if (req->requesttype & USB_DIR_IN)
  762. field |= TRB_DIR_IN;
  763. buf_64 = virt_to_phys(buffer);
  764. trb_fields[0] = lower_32_bits(buf_64);
  765. trb_fields[1] = upper_32_bits(buf_64);
  766. trb_fields[2] = length_field;
  767. trb_fields[3] = field | ep_ring->cycle_state;
  768. xhci_flush_cache((uintptr_t)buffer, length);
  769. queue_trb(ctrl, ep_ring, true, trb_fields);
  770. }
  771. /*
  772. * Queue status TRB -
  773. * see Table 7 and sections 4.11.2.2 and 6.4.1.2.3
  774. */
  775. /* If the device sent data, the status stage is an OUT transfer */
  776. field = 0;
  777. if (length > 0 && req->requesttype & USB_DIR_IN)
  778. field = 0;
  779. else
  780. field = TRB_DIR_IN;
  781. trb_fields[0] = 0;
  782. trb_fields[1] = 0;
  783. trb_fields[2] = TRB_INTR_TARGET(0);
  784. /* Event on completion */
  785. trb_fields[3] = field | TRB_IOC |
  786. TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state;
  787. queue_trb(ctrl, ep_ring, false, trb_fields);
  788. giveback_first_trb(udev, ep_index, start_cycle, start_trb);
  789. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  790. if (!event)
  791. goto abort;
  792. field = le32_to_cpu(event->trans_event.flags);
  793. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  794. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  795. record_transfer_result(udev, event, length);
  796. xhci_acknowledge_event(ctrl);
  797. /* Invalidate buffer to make it available to usb-core */
  798. if (length > 0)
  799. xhci_inval_cache((uintptr_t)buffer, length);
  800. if (GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len))
  801. == COMP_SHORT_TX) {
  802. /* Short data stage, clear up additional status stage event */
  803. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  804. if (!event)
  805. goto abort;
  806. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  807. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  808. xhci_acknowledge_event(ctrl);
  809. }
  810. return (udev->status != USB_ST_NOT_PROC) ? 0 : -1;
  811. abort:
  812. debug("XHCI control transfer timed out, aborting...\n");
  813. abort_td(udev, ep_index);
  814. udev->status = USB_ST_NAK_REC;
  815. udev->act_len = 0;
  816. return -ETIMEDOUT;
  817. }