xhci-ring.c 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * USB HOST XHCI Controller stack
  4. *
  5. * Based on xHCI host controller driver in linux-kernel
  6. * by Sarah Sharp.
  7. *
  8. * Copyright (C) 2008 Intel Corp.
  9. * Author: Sarah Sharp
  10. *
  11. * Copyright (C) 2013 Samsung Electronics Co.Ltd
  12. * Authors: Vivek Gautam <gautam.vivek@samsung.com>
  13. * Vikas Sajjan <vikas.sajjan@samsung.com>
  14. */
  15. #include <common.h>
  16. #include <cpu_func.h>
  17. #include <log.h>
  18. #include <asm/byteorder.h>
  19. #include <usb.h>
  20. #include <asm/unaligned.h>
  21. #include <linux/bug.h>
  22. #include <linux/errno.h>
  23. #include <usb/xhci.h>
  24. /**
  25. * Is this TRB a link TRB or was the last TRB the last TRB in this event ring
  26. * segment? I.e. would the updated event TRB pointer step off the end of the
  27. * event seg ?
  28. *
  29. * @param ctrl Host controller data structure
  30. * @param ring pointer to the ring
  31. * @param seg poniter to the segment to which TRB belongs
  32. * @param trb poniter to the ring trb
  33. * @return 1 if this TRB a link TRB else 0
  34. */
  35. static int last_trb(struct xhci_ctrl *ctrl, struct xhci_ring *ring,
  36. struct xhci_segment *seg, union xhci_trb *trb)
  37. {
  38. if (ring == ctrl->event_ring)
  39. return trb == &seg->trbs[TRBS_PER_SEGMENT];
  40. else
  41. return TRB_TYPE_LINK_LE32(trb->link.control);
  42. }
  43. /**
  44. * Does this link TRB point to the first segment in a ring,
  45. * or was the previous TRB the last TRB on the last segment in the ERST?
  46. *
  47. * @param ctrl Host controller data structure
  48. * @param ring pointer to the ring
  49. * @param seg poniter to the segment to which TRB belongs
  50. * @param trb poniter to the ring trb
  51. * @return 1 if this TRB is the last TRB on the last segment else 0
  52. */
  53. static bool last_trb_on_last_seg(struct xhci_ctrl *ctrl,
  54. struct xhci_ring *ring,
  55. struct xhci_segment *seg,
  56. union xhci_trb *trb)
  57. {
  58. if (ring == ctrl->event_ring)
  59. return ((trb == &seg->trbs[TRBS_PER_SEGMENT]) &&
  60. (seg->next == ring->first_seg));
  61. else
  62. return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
  63. }
  64. /**
  65. * See Cycle bit rules. SW is the consumer for the event ring only.
  66. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  67. *
  68. * If we've just enqueued a TRB that is in the middle of a TD (meaning the
  69. * chain bit is set), then set the chain bit in all the following link TRBs.
  70. * If we've enqueued the last TRB in a TD, make sure the following link TRBs
  71. * have their chain bit cleared (so that each Link TRB is a separate TD).
  72. *
  73. * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
  74. * set, but other sections talk about dealing with the chain bit set. This was
  75. * fixed in the 0.96 specification errata, but we have to assume that all 0.95
  76. * xHCI hardware can't handle the chain bit being cleared on a link TRB.
  77. *
  78. * @param ctrl Host controller data structure
  79. * @param ring pointer to the ring
  80. * @param more_trbs_coming flag to indicate whether more trbs
  81. * are expected or NOT.
  82. * Will you enqueue more TRBs before calling
  83. * prepare_ring()?
  84. * @return none
  85. */
  86. static void inc_enq(struct xhci_ctrl *ctrl, struct xhci_ring *ring,
  87. bool more_trbs_coming)
  88. {
  89. u32 chain;
  90. union xhci_trb *next;
  91. chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
  92. next = ++(ring->enqueue);
  93. /*
  94. * Update the dequeue pointer further if that was a link TRB or we're at
  95. * the end of an event ring segment (which doesn't have link TRBS)
  96. */
  97. while (last_trb(ctrl, ring, ring->enq_seg, next)) {
  98. if (ring != ctrl->event_ring) {
  99. /*
  100. * If the caller doesn't plan on enqueueing more
  101. * TDs before ringing the doorbell, then we
  102. * don't want to give the link TRB to the
  103. * hardware just yet. We'll give the link TRB
  104. * back in prepare_ring() just before we enqueue
  105. * the TD at the top of the ring.
  106. */
  107. if (!chain && !more_trbs_coming)
  108. break;
  109. /*
  110. * If we're not dealing with 0.95 hardware or
  111. * isoc rings on AMD 0.96 host,
  112. * carry over the chain bit of the previous TRB
  113. * (which may mean the chain bit is cleared).
  114. */
  115. next->link.control &= cpu_to_le32(~TRB_CHAIN);
  116. next->link.control |= cpu_to_le32(chain);
  117. next->link.control ^= cpu_to_le32(TRB_CYCLE);
  118. xhci_flush_cache((uintptr_t)next,
  119. sizeof(union xhci_trb));
  120. }
  121. /* Toggle the cycle bit after the last ring segment. */
  122. if (last_trb_on_last_seg(ctrl, ring,
  123. ring->enq_seg, next))
  124. ring->cycle_state = (ring->cycle_state ? 0 : 1);
  125. ring->enq_seg = ring->enq_seg->next;
  126. ring->enqueue = ring->enq_seg->trbs;
  127. next = ring->enqueue;
  128. }
  129. }
  130. /**
  131. * See Cycle bit rules. SW is the consumer for the event ring only.
  132. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  133. *
  134. * @param ctrl Host controller data structure
  135. * @param ring Ring whose Dequeue TRB pointer needs to be incremented.
  136. * return none
  137. */
  138. static void inc_deq(struct xhci_ctrl *ctrl, struct xhci_ring *ring)
  139. {
  140. do {
  141. /*
  142. * Update the dequeue pointer further if that was a link TRB or
  143. * we're at the end of an event ring segment (which doesn't have
  144. * link TRBS)
  145. */
  146. if (last_trb(ctrl, ring, ring->deq_seg, ring->dequeue)) {
  147. if (ring == ctrl->event_ring &&
  148. last_trb_on_last_seg(ctrl, ring,
  149. ring->deq_seg, ring->dequeue)) {
  150. ring->cycle_state = (ring->cycle_state ? 0 : 1);
  151. }
  152. ring->deq_seg = ring->deq_seg->next;
  153. ring->dequeue = ring->deq_seg->trbs;
  154. } else {
  155. ring->dequeue++;
  156. }
  157. } while (last_trb(ctrl, ring, ring->deq_seg, ring->dequeue));
  158. }
  159. /**
  160. * Generic function for queueing a TRB on a ring.
  161. * The caller must have checked to make sure there's room on the ring.
  162. *
  163. * @param more_trbs_coming: Will you enqueue more TRBs before calling
  164. * prepare_ring()?
  165. * @param ctrl Host controller data structure
  166. * @param ring pointer to the ring
  167. * @param more_trbs_coming flag to indicate whether more trbs
  168. * @param trb_fields pointer to trb field array containing TRB contents
  169. * @return pointer to the enqueued trb
  170. */
  171. static struct xhci_generic_trb *queue_trb(struct xhci_ctrl *ctrl,
  172. struct xhci_ring *ring,
  173. bool more_trbs_coming,
  174. unsigned int *trb_fields)
  175. {
  176. struct xhci_generic_trb *trb;
  177. int i;
  178. trb = &ring->enqueue->generic;
  179. for (i = 0; i < 4; i++)
  180. trb->field[i] = cpu_to_le32(trb_fields[i]);
  181. xhci_flush_cache((uintptr_t)trb, sizeof(struct xhci_generic_trb));
  182. inc_enq(ctrl, ring, more_trbs_coming);
  183. return trb;
  184. }
  185. /**
  186. * Does various checks on the endpoint ring, and makes it ready
  187. * to queue num_trbs.
  188. *
  189. * @param ctrl Host controller data structure
  190. * @param ep_ring pointer to the EP Transfer Ring
  191. * @param ep_state State of the End Point
  192. * @return error code in case of invalid ep_state, 0 on success
  193. */
  194. static int prepare_ring(struct xhci_ctrl *ctrl, struct xhci_ring *ep_ring,
  195. u32 ep_state)
  196. {
  197. union xhci_trb *next = ep_ring->enqueue;
  198. /* Make sure the endpoint has been added to xHC schedule */
  199. switch (ep_state) {
  200. case EP_STATE_DISABLED:
  201. /*
  202. * USB core changed config/interfaces without notifying us,
  203. * or hardware is reporting the wrong state.
  204. */
  205. puts("WARN urb submitted to disabled ep\n");
  206. return -ENOENT;
  207. case EP_STATE_ERROR:
  208. puts("WARN waiting for error on ep to be cleared\n");
  209. return -EINVAL;
  210. case EP_STATE_HALTED:
  211. puts("WARN halted endpoint, queueing URB anyway.\n");
  212. case EP_STATE_STOPPED:
  213. case EP_STATE_RUNNING:
  214. debug("EP STATE RUNNING.\n");
  215. break;
  216. default:
  217. puts("ERROR unknown endpoint state for ep\n");
  218. return -EINVAL;
  219. }
  220. while (last_trb(ctrl, ep_ring, ep_ring->enq_seg, next)) {
  221. /*
  222. * If we're not dealing with 0.95 hardware or isoc rings
  223. * on AMD 0.96 host, clear the chain bit.
  224. */
  225. next->link.control &= cpu_to_le32(~TRB_CHAIN);
  226. next->link.control ^= cpu_to_le32(TRB_CYCLE);
  227. xhci_flush_cache((uintptr_t)next, sizeof(union xhci_trb));
  228. /* Toggle the cycle bit after the last ring segment. */
  229. if (last_trb_on_last_seg(ctrl, ep_ring,
  230. ep_ring->enq_seg, next))
  231. ep_ring->cycle_state = (ep_ring->cycle_state ? 0 : 1);
  232. ep_ring->enq_seg = ep_ring->enq_seg->next;
  233. ep_ring->enqueue = ep_ring->enq_seg->trbs;
  234. next = ep_ring->enqueue;
  235. }
  236. return 0;
  237. }
  238. /**
  239. * Generic function for queueing a command TRB on the command ring.
  240. * Check to make sure there's room on the command ring for one command TRB.
  241. *
  242. * @param ctrl Host controller data structure
  243. * @param ptr Pointer address to write in the first two fields (opt.)
  244. * @param slot_id Slot ID to encode in the flags field (opt.)
  245. * @param ep_index Endpoint index to encode in the flags field (opt.)
  246. * @param cmd Command type to enqueue
  247. * @return none
  248. */
  249. void xhci_queue_command(struct xhci_ctrl *ctrl, u8 *ptr, u32 slot_id,
  250. u32 ep_index, trb_type cmd)
  251. {
  252. u32 fields[4];
  253. u64 val_64 = 0;
  254. BUG_ON(prepare_ring(ctrl, ctrl->cmd_ring, EP_STATE_RUNNING));
  255. if (ptr)
  256. val_64 = xhci_virt_to_bus(ctrl, ptr);
  257. fields[0] = lower_32_bits(val_64);
  258. fields[1] = upper_32_bits(val_64);
  259. fields[2] = 0;
  260. fields[3] = TRB_TYPE(cmd) | SLOT_ID_FOR_TRB(slot_id) |
  261. ctrl->cmd_ring->cycle_state;
  262. /*
  263. * Only 'reset endpoint', 'stop endpoint' and 'set TR dequeue pointer'
  264. * commands need endpoint id encoded.
  265. */
  266. if (cmd >= TRB_RESET_EP && cmd <= TRB_SET_DEQ)
  267. fields[3] |= EP_ID_FOR_TRB(ep_index);
  268. queue_trb(ctrl, ctrl->cmd_ring, false, fields);
  269. /* Ring the command ring doorbell */
  270. xhci_writel(&ctrl->dba->doorbell[0], DB_VALUE_HOST);
  271. }
  272. /*
  273. * For xHCI 1.0 host controllers, TD size is the number of max packet sized
  274. * packets remaining in the TD (*not* including this TRB).
  275. *
  276. * Total TD packet count = total_packet_count =
  277. * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
  278. *
  279. * Packets transferred up to and including this TRB = packets_transferred =
  280. * rounddown(total bytes transferred including this TRB / wMaxPacketSize)
  281. *
  282. * TD size = total_packet_count - packets_transferred
  283. *
  284. * For xHCI 0.96 and older, TD size field should be the remaining bytes
  285. * including this TRB, right shifted by 10
  286. *
  287. * For all hosts it must fit in bits 21:17, so it can't be bigger than 31.
  288. * This is taken care of in the TRB_TD_SIZE() macro
  289. *
  290. * The last TRB in a TD must have the TD size set to zero.
  291. *
  292. * @param ctrl host controller data structure
  293. * @param transferred total size sent so far
  294. * @param trb_buff_len length of the TRB Buffer
  295. * @param td_total_len total packet count
  296. * @param maxp max packet size of current pipe
  297. * @param more_trbs_coming indicate last trb in TD
  298. * @return remainder
  299. */
  300. static u32 xhci_td_remainder(struct xhci_ctrl *ctrl, int transferred,
  301. int trb_buff_len, unsigned int td_total_len,
  302. int maxp, bool more_trbs_coming)
  303. {
  304. u32 total_packet_count;
  305. /* MTK xHCI 0.96 contains some features from 1.0 */
  306. if (ctrl->hci_version < 0x100 && !(ctrl->quirks & XHCI_MTK_HOST))
  307. return ((td_total_len - transferred) >> 10);
  308. /* One TRB with a zero-length data packet. */
  309. if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
  310. trb_buff_len == td_total_len)
  311. return 0;
  312. /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */
  313. if ((ctrl->quirks & XHCI_MTK_HOST) && (ctrl->hci_version < 0x100))
  314. trb_buff_len = 0;
  315. total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
  316. /* Queueing functions don't count the current TRB into transferred */
  317. return (total_packet_count - ((transferred + trb_buff_len) / maxp));
  318. }
  319. /**
  320. * Ring the doorbell of the End Point
  321. *
  322. * @param udev pointer to the USB device structure
  323. * @param ep_index index of the endpoint
  324. * @param start_cycle cycle flag of the first TRB
  325. * @param start_trb pionter to the first TRB
  326. * @return none
  327. */
  328. static void giveback_first_trb(struct usb_device *udev, int ep_index,
  329. int start_cycle,
  330. struct xhci_generic_trb *start_trb)
  331. {
  332. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  333. /*
  334. * Pass all the TRBs to the hardware at once and make sure this write
  335. * isn't reordered.
  336. */
  337. if (start_cycle)
  338. start_trb->field[3] |= cpu_to_le32(start_cycle);
  339. else
  340. start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
  341. xhci_flush_cache((uintptr_t)start_trb, sizeof(struct xhci_generic_trb));
  342. /* Ringing EP doorbell here */
  343. xhci_writel(&ctrl->dba->doorbell[udev->slot_id],
  344. DB_VALUE(ep_index, 0));
  345. return;
  346. }
  347. /**** POLLING mechanism for XHCI ****/
  348. /**
  349. * Finalizes a handled event TRB by advancing our dequeue pointer and giving
  350. * the TRB back to the hardware for recycling. Must call this exactly once at
  351. * the end of each event handler, and not touch the TRB again afterwards.
  352. *
  353. * @param ctrl Host controller data structure
  354. * @return none
  355. */
  356. void xhci_acknowledge_event(struct xhci_ctrl *ctrl)
  357. {
  358. /* Advance our dequeue pointer to the next event */
  359. inc_deq(ctrl, ctrl->event_ring);
  360. /* Inform the hardware */
  361. xhci_writeq(&ctrl->ir_set->erst_dequeue,
  362. xhci_virt_to_bus(ctrl, ctrl->event_ring->dequeue) | ERST_EHB);
  363. }
  364. /**
  365. * Checks if there is a new event to handle on the event ring.
  366. *
  367. * @param ctrl Host controller data structure
  368. * @return 0 if failure else 1 on success
  369. */
  370. static int event_ready(struct xhci_ctrl *ctrl)
  371. {
  372. union xhci_trb *event;
  373. xhci_inval_cache((uintptr_t)ctrl->event_ring->dequeue,
  374. sizeof(union xhci_trb));
  375. event = ctrl->event_ring->dequeue;
  376. /* Does the HC or OS own the TRB? */
  377. if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
  378. ctrl->event_ring->cycle_state)
  379. return 0;
  380. return 1;
  381. }
  382. /**
  383. * Waits for a specific type of event and returns it. Discards unexpected
  384. * events. Caller *must* call xhci_acknowledge_event() after it is finished
  385. * processing the event, and must not access the returned pointer afterwards.
  386. *
  387. * @param ctrl Host controller data structure
  388. * @param expected TRB type expected from Event TRB
  389. * @return pointer to event trb
  390. */
  391. union xhci_trb *xhci_wait_for_event(struct xhci_ctrl *ctrl, trb_type expected)
  392. {
  393. trb_type type;
  394. unsigned long ts = get_timer(0);
  395. do {
  396. union xhci_trb *event = ctrl->event_ring->dequeue;
  397. if (!event_ready(ctrl))
  398. continue;
  399. type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags));
  400. if (type == expected)
  401. return event;
  402. if (type == TRB_PORT_STATUS)
  403. /* TODO: remove this once enumeration has been reworked */
  404. /*
  405. * Port status change events always have a
  406. * successful completion code
  407. */
  408. BUG_ON(GET_COMP_CODE(
  409. le32_to_cpu(event->generic.field[2])) !=
  410. COMP_SUCCESS);
  411. else
  412. printf("Unexpected XHCI event TRB, skipping... "
  413. "(%08x %08x %08x %08x)\n",
  414. le32_to_cpu(event->generic.field[0]),
  415. le32_to_cpu(event->generic.field[1]),
  416. le32_to_cpu(event->generic.field[2]),
  417. le32_to_cpu(event->generic.field[3]));
  418. xhci_acknowledge_event(ctrl);
  419. } while (get_timer(ts) < XHCI_TIMEOUT);
  420. if (expected == TRB_TRANSFER)
  421. return NULL;
  422. printf("XHCI timeout on event type %d... cannot recover.\n", expected);
  423. BUG();
  424. }
  425. /*
  426. * Stops transfer processing for an endpoint and throws away all unprocessed
  427. * TRBs by setting the xHC's dequeue pointer to our enqueue pointer. The next
  428. * xhci_bulk_tx/xhci_ctrl_tx on this enpoint will add new transfers there and
  429. * ring the doorbell, causing this endpoint to start working again.
  430. * (Careful: This will BUG() when there was no transfer in progress. Shouldn't
  431. * happen in practice for current uses and is too complicated to fix right now.)
  432. */
  433. static void abort_td(struct usb_device *udev, int ep_index)
  434. {
  435. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  436. struct xhci_ring *ring = ctrl->devs[udev->slot_id]->eps[ep_index].ring;
  437. union xhci_trb *event;
  438. u32 field;
  439. xhci_queue_command(ctrl, NULL, udev->slot_id, ep_index, TRB_STOP_RING);
  440. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  441. if (!event)
  442. return;
  443. field = le32_to_cpu(event->trans_event.flags);
  444. BUG_ON(TRB_TO_SLOT_ID(field) != udev->slot_id);
  445. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  446. BUG_ON(GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len
  447. != COMP_STOP)));
  448. xhci_acknowledge_event(ctrl);
  449. event = xhci_wait_for_event(ctrl, TRB_COMPLETION);
  450. BUG_ON(TRB_TO_SLOT_ID(le32_to_cpu(event->event_cmd.flags))
  451. != udev->slot_id || GET_COMP_CODE(le32_to_cpu(
  452. event->event_cmd.status)) != COMP_SUCCESS);
  453. xhci_acknowledge_event(ctrl);
  454. xhci_queue_command(ctrl, (void *)((uintptr_t)ring->enqueue |
  455. ring->cycle_state), udev->slot_id, ep_index, TRB_SET_DEQ);
  456. event = xhci_wait_for_event(ctrl, TRB_COMPLETION);
  457. BUG_ON(TRB_TO_SLOT_ID(le32_to_cpu(event->event_cmd.flags))
  458. != udev->slot_id || GET_COMP_CODE(le32_to_cpu(
  459. event->event_cmd.status)) != COMP_SUCCESS);
  460. xhci_acknowledge_event(ctrl);
  461. }
  462. static void record_transfer_result(struct usb_device *udev,
  463. union xhci_trb *event, int length)
  464. {
  465. udev->act_len = min(length, length -
  466. (int)EVENT_TRB_LEN(le32_to_cpu(event->trans_event.transfer_len)));
  467. switch (GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len))) {
  468. case COMP_SUCCESS:
  469. BUG_ON(udev->act_len != length);
  470. /* fallthrough */
  471. case COMP_SHORT_TX:
  472. udev->status = 0;
  473. break;
  474. case COMP_STALL:
  475. udev->status = USB_ST_STALLED;
  476. break;
  477. case COMP_DB_ERR:
  478. case COMP_TRB_ERR:
  479. udev->status = USB_ST_BUF_ERR;
  480. break;
  481. case COMP_BABBLE:
  482. udev->status = USB_ST_BABBLE_DET;
  483. break;
  484. default:
  485. udev->status = 0x80; /* USB_ST_TOO_LAZY_TO_MAKE_A_NEW_MACRO */
  486. }
  487. }
  488. /**** Bulk and Control transfer methods ****/
  489. /**
  490. * Queues up the BULK Request
  491. *
  492. * @param udev pointer to the USB device structure
  493. * @param pipe contains the DIR_IN or OUT , devnum
  494. * @param length length of the buffer
  495. * @param buffer buffer to be read/written based on the request
  496. * @return returns 0 if successful else -1 on failure
  497. */
  498. int xhci_bulk_tx(struct usb_device *udev, unsigned long pipe,
  499. int length, void *buffer)
  500. {
  501. int num_trbs = 0;
  502. struct xhci_generic_trb *start_trb;
  503. bool first_trb = false;
  504. int start_cycle;
  505. u32 field = 0;
  506. u32 length_field = 0;
  507. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  508. int slot_id = udev->slot_id;
  509. int ep_index;
  510. struct xhci_virt_device *virt_dev;
  511. struct xhci_ep_ctx *ep_ctx;
  512. struct xhci_ring *ring; /* EP transfer ring */
  513. union xhci_trb *event;
  514. int running_total, trb_buff_len;
  515. bool more_trbs_coming = true;
  516. int maxpacketsize;
  517. u64 addr;
  518. int ret;
  519. u32 trb_fields[4];
  520. u64 val_64 = xhci_virt_to_bus(ctrl, buffer);
  521. void *last_transfer_trb_addr;
  522. int available_length;
  523. debug("dev=%p, pipe=%lx, buffer=%p, length=%d\n",
  524. udev, pipe, buffer, length);
  525. available_length = length;
  526. ep_index = usb_pipe_ep_index(pipe);
  527. virt_dev = ctrl->devs[slot_id];
  528. xhci_inval_cache((uintptr_t)virt_dev->out_ctx->bytes,
  529. virt_dev->out_ctx->size);
  530. ep_ctx = xhci_get_ep_ctx(ctrl, virt_dev->out_ctx, ep_index);
  531. ring = virt_dev->eps[ep_index].ring;
  532. /*
  533. * How much data is (potentially) left before the 64KB boundary?
  534. * XHCI Spec puts restriction( TABLE 49 and 6.4.1 section of XHCI Spec)
  535. * that the buffer should not span 64KB boundary. if so
  536. * we send request in more than 1 TRB by chaining them.
  537. */
  538. running_total = TRB_MAX_BUFF_SIZE -
  539. (lower_32_bits(val_64) & (TRB_MAX_BUFF_SIZE - 1));
  540. trb_buff_len = running_total;
  541. running_total &= TRB_MAX_BUFF_SIZE - 1;
  542. /*
  543. * If there's some data on this 64KB chunk, or we have to send a
  544. * zero-length transfer, we need at least one TRB
  545. */
  546. if (running_total != 0 || length == 0)
  547. num_trbs++;
  548. /* How many more 64KB chunks to transfer, how many more TRBs? */
  549. while (running_total < length) {
  550. num_trbs++;
  551. running_total += TRB_MAX_BUFF_SIZE;
  552. }
  553. /*
  554. * XXX: Calling routine prepare_ring() called in place of
  555. * prepare_trasfer() as there in 'Linux' since we are not
  556. * maintaining multiple TDs/transfer at the same time.
  557. */
  558. ret = prepare_ring(ctrl, ring,
  559. le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK);
  560. if (ret < 0)
  561. return ret;
  562. /*
  563. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  564. * until we've finished creating all the other TRBs. The ring's cycle
  565. * state may change as we enqueue the other TRBs, so save it too.
  566. */
  567. start_trb = &ring->enqueue->generic;
  568. start_cycle = ring->cycle_state;
  569. running_total = 0;
  570. maxpacketsize = usb_maxpacket(udev, pipe);
  571. /* How much data is in the first TRB? */
  572. /*
  573. * How much data is (potentially) left before the 64KB boundary?
  574. * XHCI Spec puts restriction( TABLE 49 and 6.4.1 section of XHCI Spec)
  575. * that the buffer should not span 64KB boundary. if so
  576. * we send request in more than 1 TRB by chaining them.
  577. */
  578. addr = val_64;
  579. if (trb_buff_len > length)
  580. trb_buff_len = length;
  581. first_trb = true;
  582. /* flush the buffer before use */
  583. xhci_flush_cache((uintptr_t)buffer, length);
  584. /* Queue the first TRB, even if it's zero-length */
  585. do {
  586. u32 remainder = 0;
  587. field = 0;
  588. /* Don't change the cycle bit of the first TRB until later */
  589. if (first_trb) {
  590. first_trb = false;
  591. if (start_cycle == 0)
  592. field |= TRB_CYCLE;
  593. } else {
  594. field |= ring->cycle_state;
  595. }
  596. /*
  597. * Chain all the TRBs together; clear the chain bit in the last
  598. * TRB to indicate it's the last TRB in the chain.
  599. */
  600. if (num_trbs > 1) {
  601. field |= TRB_CHAIN;
  602. } else {
  603. field |= TRB_IOC;
  604. more_trbs_coming = false;
  605. }
  606. /* Only set interrupt on short packet for IN endpoints */
  607. if (usb_pipein(pipe))
  608. field |= TRB_ISP;
  609. /* Set the TRB length, TD size, and interrupter fields. */
  610. remainder = xhci_td_remainder(ctrl, running_total, trb_buff_len,
  611. length, maxpacketsize,
  612. more_trbs_coming);
  613. length_field = (TRB_LEN(trb_buff_len) |
  614. TRB_TD_SIZE(remainder) |
  615. TRB_INTR_TARGET(0));
  616. trb_fields[0] = lower_32_bits(addr);
  617. trb_fields[1] = upper_32_bits(addr);
  618. trb_fields[2] = length_field;
  619. trb_fields[3] = field | TRB_TYPE(TRB_NORMAL);
  620. last_transfer_trb_addr = queue_trb(ctrl, ring, (num_trbs > 1), trb_fields);
  621. --num_trbs;
  622. running_total += trb_buff_len;
  623. /* Calculate length for next transfer */
  624. addr += trb_buff_len;
  625. trb_buff_len = min((length - running_total), TRB_MAX_BUFF_SIZE);
  626. } while (running_total < length);
  627. giveback_first_trb(udev, ep_index, start_cycle, start_trb);
  628. again:
  629. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  630. if (!event) {
  631. debug("XHCI bulk transfer timed out, aborting...\n");
  632. abort_td(udev, ep_index);
  633. udev->status = USB_ST_NAK_REC; /* closest thing to a timeout */
  634. udev->act_len = 0;
  635. return -ETIMEDOUT;
  636. }
  637. if ((uintptr_t)(le64_to_cpu(event->trans_event.buffer)) !=
  638. (uintptr_t)xhci_virt_to_bus(ctrl, last_transfer_trb_addr)) {
  639. available_length -=
  640. (int)EVENT_TRB_LEN(le32_to_cpu(event->trans_event.transfer_len));
  641. xhci_acknowledge_event(ctrl);
  642. goto again;
  643. }
  644. field = le32_to_cpu(event->trans_event.flags);
  645. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  646. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  647. record_transfer_result(udev, event, available_length);
  648. xhci_acknowledge_event(ctrl);
  649. xhci_inval_cache((uintptr_t)buffer, length);
  650. return (udev->status != USB_ST_NOT_PROC) ? 0 : -1;
  651. }
  652. /**
  653. * Queues up the Control Transfer Request
  654. *
  655. * @param udev pointer to the USB device structure
  656. * @param pipe contains the DIR_IN or OUT , devnum
  657. * @param req request type
  658. * @param length length of the buffer
  659. * @param buffer buffer to be read/written based on the request
  660. * @return returns 0 if successful else error code on failure
  661. */
  662. int xhci_ctrl_tx(struct usb_device *udev, unsigned long pipe,
  663. struct devrequest *req, int length,
  664. void *buffer)
  665. {
  666. int ret;
  667. int start_cycle;
  668. int num_trbs;
  669. u32 field;
  670. u32 length_field;
  671. u64 buf_64 = 0;
  672. struct xhci_generic_trb *start_trb;
  673. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  674. int slot_id = udev->slot_id;
  675. int ep_index;
  676. u32 trb_fields[4];
  677. struct xhci_virt_device *virt_dev = ctrl->devs[slot_id];
  678. struct xhci_ring *ep_ring;
  679. union xhci_trb *event;
  680. u32 remainder;
  681. debug("req=%u (%#x), type=%u (%#x), value=%u (%#x), index=%u\n",
  682. req->request, req->request,
  683. req->requesttype, req->requesttype,
  684. le16_to_cpu(req->value), le16_to_cpu(req->value),
  685. le16_to_cpu(req->index));
  686. ep_index = usb_pipe_ep_index(pipe);
  687. ep_ring = virt_dev->eps[ep_index].ring;
  688. /*
  689. * Check to see if the max packet size for the default control
  690. * endpoint changed during FS device enumeration
  691. */
  692. if (udev->speed == USB_SPEED_FULL) {
  693. ret = xhci_check_maxpacket(udev);
  694. if (ret < 0)
  695. return ret;
  696. }
  697. xhci_inval_cache((uintptr_t)virt_dev->out_ctx->bytes,
  698. virt_dev->out_ctx->size);
  699. struct xhci_ep_ctx *ep_ctx = NULL;
  700. ep_ctx = xhci_get_ep_ctx(ctrl, virt_dev->out_ctx, ep_index);
  701. /* 1 TRB for setup, 1 for status */
  702. num_trbs = 2;
  703. /*
  704. * Don't need to check if we need additional event data and normal TRBs,
  705. * since data in control transfers will never get bigger than 16MB
  706. * XXX: can we get a buffer that crosses 64KB boundaries?
  707. */
  708. if (length > 0)
  709. num_trbs++;
  710. /*
  711. * XXX: Calling routine prepare_ring() called in place of
  712. * prepare_trasfer() as there in 'Linux' since we are not
  713. * maintaining multiple TDs/transfer at the same time.
  714. */
  715. ret = prepare_ring(ctrl, ep_ring,
  716. le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK);
  717. if (ret < 0)
  718. return ret;
  719. /*
  720. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  721. * until we've finished creating all the other TRBs. The ring's cycle
  722. * state may change as we enqueue the other TRBs, so save it too.
  723. */
  724. start_trb = &ep_ring->enqueue->generic;
  725. start_cycle = ep_ring->cycle_state;
  726. debug("start_trb %p, start_cycle %d\n", start_trb, start_cycle);
  727. /* Queue setup TRB - see section 6.4.1.2.1 */
  728. /* FIXME better way to translate setup_packet into two u32 fields? */
  729. field = 0;
  730. field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
  731. if (start_cycle == 0)
  732. field |= 0x1;
  733. /* xHCI 1.0 6.4.1.2.1: Transfer Type field */
  734. if (ctrl->hci_version >= 0x100 || ctrl->quirks & XHCI_MTK_HOST) {
  735. if (length > 0) {
  736. if (req->requesttype & USB_DIR_IN)
  737. field |= TRB_TX_TYPE(TRB_DATA_IN);
  738. else
  739. field |= TRB_TX_TYPE(TRB_DATA_OUT);
  740. }
  741. }
  742. debug("req->requesttype = %d, req->request = %d, req->value = %d, req->index = %d, req->length = %d\n",
  743. req->requesttype, req->request, le16_to_cpu(req->value),
  744. le16_to_cpu(req->index), le16_to_cpu(req->length));
  745. trb_fields[0] = req->requesttype | req->request << 8 |
  746. le16_to_cpu(req->value) << 16;
  747. trb_fields[1] = le16_to_cpu(req->index) |
  748. le16_to_cpu(req->length) << 16;
  749. /* TRB_LEN | (TRB_INTR_TARGET) */
  750. trb_fields[2] = (TRB_LEN(8) | TRB_INTR_TARGET(0));
  751. /* Immediate data in pointer */
  752. trb_fields[3] = field;
  753. queue_trb(ctrl, ep_ring, true, trb_fields);
  754. /* Re-initializing field to zero */
  755. field = 0;
  756. /* If there's data, queue data TRBs */
  757. /* Only set interrupt on short packet for IN endpoints */
  758. if (usb_pipein(pipe))
  759. field = TRB_ISP | TRB_TYPE(TRB_DATA);
  760. else
  761. field = TRB_TYPE(TRB_DATA);
  762. remainder = xhci_td_remainder(ctrl, 0, length, length,
  763. usb_maxpacket(udev, pipe), true);
  764. length_field = TRB_LEN(length) | TRB_TD_SIZE(remainder) |
  765. TRB_INTR_TARGET(0);
  766. debug("length_field = %d, length = %d,"
  767. "xhci_td_remainder(length) = %d , TRB_INTR_TARGET(0) = %d\n",
  768. length_field, TRB_LEN(length),
  769. TRB_TD_SIZE(remainder), 0);
  770. if (length > 0) {
  771. if (req->requesttype & USB_DIR_IN)
  772. field |= TRB_DIR_IN;
  773. buf_64 = xhci_virt_to_bus(ctrl, buffer);
  774. trb_fields[0] = lower_32_bits(buf_64);
  775. trb_fields[1] = upper_32_bits(buf_64);
  776. trb_fields[2] = length_field;
  777. trb_fields[3] = field | ep_ring->cycle_state;
  778. xhci_flush_cache((uintptr_t)buffer, length);
  779. queue_trb(ctrl, ep_ring, true, trb_fields);
  780. }
  781. /*
  782. * Queue status TRB -
  783. * see Table 7 and sections 4.11.2.2 and 6.4.1.2.3
  784. */
  785. /* If the device sent data, the status stage is an OUT transfer */
  786. field = 0;
  787. if (length > 0 && req->requesttype & USB_DIR_IN)
  788. field = 0;
  789. else
  790. field = TRB_DIR_IN;
  791. trb_fields[0] = 0;
  792. trb_fields[1] = 0;
  793. trb_fields[2] = TRB_INTR_TARGET(0);
  794. /* Event on completion */
  795. trb_fields[3] = field | TRB_IOC |
  796. TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state;
  797. queue_trb(ctrl, ep_ring, false, trb_fields);
  798. giveback_first_trb(udev, ep_index, start_cycle, start_trb);
  799. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  800. if (!event)
  801. goto abort;
  802. field = le32_to_cpu(event->trans_event.flags);
  803. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  804. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  805. record_transfer_result(udev, event, length);
  806. xhci_acknowledge_event(ctrl);
  807. /* Invalidate buffer to make it available to usb-core */
  808. if (length > 0)
  809. xhci_inval_cache((uintptr_t)buffer, length);
  810. if (GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len))
  811. == COMP_SHORT_TX) {
  812. /* Short data stage, clear up additional status stage event */
  813. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  814. if (!event)
  815. goto abort;
  816. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  817. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  818. xhci_acknowledge_event(ctrl);
  819. }
  820. return (udev->status != USB_ST_NOT_PROC) ? 0 : -1;
  821. abort:
  822. debug("XHCI control transfer timed out, aborting...\n");
  823. abort_td(udev, ep_index);
  824. udev->status = USB_ST_NAK_REC;
  825. udev->act_len = 0;
  826. return -ETIMEDOUT;
  827. }