xhci-ring.c 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * USB HOST XHCI Controller stack
  4. *
  5. * Based on xHCI host controller driver in linux-kernel
  6. * by Sarah Sharp.
  7. *
  8. * Copyright (C) 2008 Intel Corp.
  9. * Author: Sarah Sharp
  10. *
  11. * Copyright (C) 2013 Samsung Electronics Co.Ltd
  12. * Authors: Vivek Gautam <gautam.vivek@samsung.com>
  13. * Vikas Sajjan <vikas.sajjan@samsung.com>
  14. */
  15. #include <common.h>
  16. #include <cpu_func.h>
  17. #include <log.h>
  18. #include <asm/byteorder.h>
  19. #include <usb.h>
  20. #include <asm/unaligned.h>
  21. #include <linux/bug.h>
  22. #include <linux/errno.h>
  23. #include <usb/xhci.h>
  24. /**
  25. * Is this TRB a link TRB or was the last TRB the last TRB in this event ring
  26. * segment? I.e. would the updated event TRB pointer step off the end of the
  27. * event seg ?
  28. *
  29. * @param ctrl Host controller data structure
  30. * @param ring pointer to the ring
  31. * @param seg poniter to the segment to which TRB belongs
  32. * @param trb poniter to the ring trb
  33. * @return 1 if this TRB a link TRB else 0
  34. */
  35. static int last_trb(struct xhci_ctrl *ctrl, struct xhci_ring *ring,
  36. struct xhci_segment *seg, union xhci_trb *trb)
  37. {
  38. if (ring == ctrl->event_ring)
  39. return trb == &seg->trbs[TRBS_PER_SEGMENT];
  40. else
  41. return TRB_TYPE_LINK_LE32(trb->link.control);
  42. }
  43. /**
  44. * Does this link TRB point to the first segment in a ring,
  45. * or was the previous TRB the last TRB on the last segment in the ERST?
  46. *
  47. * @param ctrl Host controller data structure
  48. * @param ring pointer to the ring
  49. * @param seg poniter to the segment to which TRB belongs
  50. * @param trb poniter to the ring trb
  51. * @return 1 if this TRB is the last TRB on the last segment else 0
  52. */
  53. static bool last_trb_on_last_seg(struct xhci_ctrl *ctrl,
  54. struct xhci_ring *ring,
  55. struct xhci_segment *seg,
  56. union xhci_trb *trb)
  57. {
  58. if (ring == ctrl->event_ring)
  59. return ((trb == &seg->trbs[TRBS_PER_SEGMENT]) &&
  60. (seg->next == ring->first_seg));
  61. else
  62. return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
  63. }
  64. /**
  65. * See Cycle bit rules. SW is the consumer for the event ring only.
  66. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  67. *
  68. * If we've just enqueued a TRB that is in the middle of a TD (meaning the
  69. * chain bit is set), then set the chain bit in all the following link TRBs.
  70. * If we've enqueued the last TRB in a TD, make sure the following link TRBs
  71. * have their chain bit cleared (so that each Link TRB is a separate TD).
  72. *
  73. * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
  74. * set, but other sections talk about dealing with the chain bit set. This was
  75. * fixed in the 0.96 specification errata, but we have to assume that all 0.95
  76. * xHCI hardware can't handle the chain bit being cleared on a link TRB.
  77. *
  78. * @param ctrl Host controller data structure
  79. * @param ring pointer to the ring
  80. * @param more_trbs_coming flag to indicate whether more trbs
  81. * are expected or NOT.
  82. * Will you enqueue more TRBs before calling
  83. * prepare_ring()?
  84. * @return none
  85. */
  86. static void inc_enq(struct xhci_ctrl *ctrl, struct xhci_ring *ring,
  87. bool more_trbs_coming)
  88. {
  89. u32 chain;
  90. union xhci_trb *next;
  91. chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
  92. next = ++(ring->enqueue);
  93. /*
  94. * Update the dequeue pointer further if that was a link TRB or we're at
  95. * the end of an event ring segment (which doesn't have link TRBS)
  96. */
  97. while (last_trb(ctrl, ring, ring->enq_seg, next)) {
  98. if (ring != ctrl->event_ring) {
  99. /*
  100. * If the caller doesn't plan on enqueueing more
  101. * TDs before ringing the doorbell, then we
  102. * don't want to give the link TRB to the
  103. * hardware just yet. We'll give the link TRB
  104. * back in prepare_ring() just before we enqueue
  105. * the TD at the top of the ring.
  106. */
  107. if (!chain && !more_trbs_coming)
  108. break;
  109. /*
  110. * If we're not dealing with 0.95 hardware or
  111. * isoc rings on AMD 0.96 host,
  112. * carry over the chain bit of the previous TRB
  113. * (which may mean the chain bit is cleared).
  114. */
  115. next->link.control &= cpu_to_le32(~TRB_CHAIN);
  116. next->link.control |= cpu_to_le32(chain);
  117. next->link.control ^= cpu_to_le32(TRB_CYCLE);
  118. xhci_flush_cache((uintptr_t)next,
  119. sizeof(union xhci_trb));
  120. }
  121. /* Toggle the cycle bit after the last ring segment. */
  122. if (last_trb_on_last_seg(ctrl, ring,
  123. ring->enq_seg, next))
  124. ring->cycle_state = (ring->cycle_state ? 0 : 1);
  125. ring->enq_seg = ring->enq_seg->next;
  126. ring->enqueue = ring->enq_seg->trbs;
  127. next = ring->enqueue;
  128. }
  129. }
  130. /**
  131. * See Cycle bit rules. SW is the consumer for the event ring only.
  132. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  133. *
  134. * @param ctrl Host controller data structure
  135. * @param ring Ring whose Dequeue TRB pointer needs to be incremented.
  136. * return none
  137. */
  138. static void inc_deq(struct xhci_ctrl *ctrl, struct xhci_ring *ring)
  139. {
  140. do {
  141. /*
  142. * Update the dequeue pointer further if that was a link TRB or
  143. * we're at the end of an event ring segment (which doesn't have
  144. * link TRBS)
  145. */
  146. if (last_trb(ctrl, ring, ring->deq_seg, ring->dequeue)) {
  147. if (ring == ctrl->event_ring &&
  148. last_trb_on_last_seg(ctrl, ring,
  149. ring->deq_seg, ring->dequeue)) {
  150. ring->cycle_state = (ring->cycle_state ? 0 : 1);
  151. }
  152. ring->deq_seg = ring->deq_seg->next;
  153. ring->dequeue = ring->deq_seg->trbs;
  154. } else {
  155. ring->dequeue++;
  156. }
  157. } while (last_trb(ctrl, ring, ring->deq_seg, ring->dequeue));
  158. }
  159. /**
  160. * Generic function for queueing a TRB on a ring.
  161. * The caller must have checked to make sure there's room on the ring.
  162. *
  163. * @param more_trbs_coming: Will you enqueue more TRBs before calling
  164. * prepare_ring()?
  165. * @param ctrl Host controller data structure
  166. * @param ring pointer to the ring
  167. * @param more_trbs_coming flag to indicate whether more trbs
  168. * @param trb_fields pointer to trb field array containing TRB contents
  169. * @return pointer to the enqueued trb
  170. */
  171. static struct xhci_generic_trb *queue_trb(struct xhci_ctrl *ctrl,
  172. struct xhci_ring *ring,
  173. bool more_trbs_coming,
  174. unsigned int *trb_fields)
  175. {
  176. struct xhci_generic_trb *trb;
  177. int i;
  178. trb = &ring->enqueue->generic;
  179. for (i = 0; i < 4; i++)
  180. trb->field[i] = cpu_to_le32(trb_fields[i]);
  181. xhci_flush_cache((uintptr_t)trb, sizeof(struct xhci_generic_trb));
  182. inc_enq(ctrl, ring, more_trbs_coming);
  183. return trb;
  184. }
  185. /**
  186. * Does various checks on the endpoint ring, and makes it ready
  187. * to queue num_trbs.
  188. *
  189. * @param ctrl Host controller data structure
  190. * @param ep_ring pointer to the EP Transfer Ring
  191. * @param ep_state State of the End Point
  192. * @return error code in case of invalid ep_state, 0 on success
  193. */
  194. static int prepare_ring(struct xhci_ctrl *ctrl, struct xhci_ring *ep_ring,
  195. u32 ep_state)
  196. {
  197. union xhci_trb *next = ep_ring->enqueue;
  198. /* Make sure the endpoint has been added to xHC schedule */
  199. switch (ep_state) {
  200. case EP_STATE_DISABLED:
  201. /*
  202. * USB core changed config/interfaces without notifying us,
  203. * or hardware is reporting the wrong state.
  204. */
  205. puts("WARN urb submitted to disabled ep\n");
  206. return -ENOENT;
  207. case EP_STATE_ERROR:
  208. puts("WARN waiting for error on ep to be cleared\n");
  209. return -EINVAL;
  210. case EP_STATE_HALTED:
  211. puts("WARN halted endpoint, queueing URB anyway.\n");
  212. case EP_STATE_STOPPED:
  213. case EP_STATE_RUNNING:
  214. debug("EP STATE RUNNING.\n");
  215. break;
  216. default:
  217. puts("ERROR unknown endpoint state for ep\n");
  218. return -EINVAL;
  219. }
  220. while (last_trb(ctrl, ep_ring, ep_ring->enq_seg, next)) {
  221. /*
  222. * If we're not dealing with 0.95 hardware or isoc rings
  223. * on AMD 0.96 host, clear the chain bit.
  224. */
  225. next->link.control &= cpu_to_le32(~TRB_CHAIN);
  226. next->link.control ^= cpu_to_le32(TRB_CYCLE);
  227. xhci_flush_cache((uintptr_t)next, sizeof(union xhci_trb));
  228. /* Toggle the cycle bit after the last ring segment. */
  229. if (last_trb_on_last_seg(ctrl, ep_ring,
  230. ep_ring->enq_seg, next))
  231. ep_ring->cycle_state = (ep_ring->cycle_state ? 0 : 1);
  232. ep_ring->enq_seg = ep_ring->enq_seg->next;
  233. ep_ring->enqueue = ep_ring->enq_seg->trbs;
  234. next = ep_ring->enqueue;
  235. }
  236. return 0;
  237. }
  238. /**
  239. * Generic function for queueing a command TRB on the command ring.
  240. * Check to make sure there's room on the command ring for one command TRB.
  241. *
  242. * @param ctrl Host controller data structure
  243. * @param ptr Pointer address to write in the first two fields (opt.)
  244. * @param slot_id Slot ID to encode in the flags field (opt.)
  245. * @param ep_index Endpoint index to encode in the flags field (opt.)
  246. * @param cmd Command type to enqueue
  247. * @return none
  248. */
  249. void xhci_queue_command(struct xhci_ctrl *ctrl, u8 *ptr, u32 slot_id,
  250. u32 ep_index, trb_type cmd)
  251. {
  252. u32 fields[4];
  253. u64 val_64 = (uintptr_t)ptr;
  254. BUG_ON(prepare_ring(ctrl, ctrl->cmd_ring, EP_STATE_RUNNING));
  255. fields[0] = lower_32_bits(val_64);
  256. fields[1] = upper_32_bits(val_64);
  257. fields[2] = 0;
  258. fields[3] = TRB_TYPE(cmd) | SLOT_ID_FOR_TRB(slot_id) |
  259. ctrl->cmd_ring->cycle_state;
  260. /*
  261. * Only 'reset endpoint', 'stop endpoint' and 'set TR dequeue pointer'
  262. * commands need endpoint id encoded.
  263. */
  264. if (cmd >= TRB_RESET_EP && cmd <= TRB_SET_DEQ)
  265. fields[3] |= EP_ID_FOR_TRB(ep_index);
  266. queue_trb(ctrl, ctrl->cmd_ring, false, fields);
  267. /* Ring the command ring doorbell */
  268. xhci_writel(&ctrl->dba->doorbell[0], DB_VALUE_HOST);
  269. }
  270. /**
  271. * The TD size is the number of bytes remaining in the TD (including this TRB),
  272. * right shifted by 10.
  273. * It must fit in bits 21:17, so it can't be bigger than 31.
  274. *
  275. * @param remainder remaining packets to be sent
  276. * @return remainder if remainder is less than max else max
  277. */
  278. static u32 xhci_td_remainder(unsigned int remainder)
  279. {
  280. u32 max = (1 << (21 - 17 + 1)) - 1;
  281. if ((remainder >> 10) >= max)
  282. return max << 17;
  283. else
  284. return (remainder >> 10) << 17;
  285. }
  286. /**
  287. * Finds out the remanining packets to be sent
  288. *
  289. * @param running_total total size sent so far
  290. * @param trb_buff_len length of the TRB Buffer
  291. * @param total_packet_count total packet count
  292. * @param maxpacketsize max packet size of current pipe
  293. * @param num_trbs_left number of TRBs left to be processed
  294. * @return 0 if running_total or trb_buff_len is 0, else remainder
  295. */
  296. static u32 xhci_v1_0_td_remainder(int running_total,
  297. int trb_buff_len,
  298. unsigned int total_packet_count,
  299. int maxpacketsize,
  300. unsigned int num_trbs_left)
  301. {
  302. int packets_transferred;
  303. /* One TRB with a zero-length data packet. */
  304. if (num_trbs_left == 0 || (running_total == 0 && trb_buff_len == 0))
  305. return 0;
  306. /*
  307. * All the TRB queueing functions don't count the current TRB in
  308. * running_total.
  309. */
  310. packets_transferred = (running_total + trb_buff_len) / maxpacketsize;
  311. if ((total_packet_count - packets_transferred) > 31)
  312. return 31 << 17;
  313. return (total_packet_count - packets_transferred) << 17;
  314. }
  315. /**
  316. * Ring the doorbell of the End Point
  317. *
  318. * @param udev pointer to the USB device structure
  319. * @param ep_index index of the endpoint
  320. * @param start_cycle cycle flag of the first TRB
  321. * @param start_trb pionter to the first TRB
  322. * @return none
  323. */
  324. static void giveback_first_trb(struct usb_device *udev, int ep_index,
  325. int start_cycle,
  326. struct xhci_generic_trb *start_trb)
  327. {
  328. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  329. /*
  330. * Pass all the TRBs to the hardware at once and make sure this write
  331. * isn't reordered.
  332. */
  333. if (start_cycle)
  334. start_trb->field[3] |= cpu_to_le32(start_cycle);
  335. else
  336. start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
  337. xhci_flush_cache((uintptr_t)start_trb, sizeof(struct xhci_generic_trb));
  338. /* Ringing EP doorbell here */
  339. xhci_writel(&ctrl->dba->doorbell[udev->slot_id],
  340. DB_VALUE(ep_index, 0));
  341. return;
  342. }
  343. /**** POLLING mechanism for XHCI ****/
  344. /**
  345. * Finalizes a handled event TRB by advancing our dequeue pointer and giving
  346. * the TRB back to the hardware for recycling. Must call this exactly once at
  347. * the end of each event handler, and not touch the TRB again afterwards.
  348. *
  349. * @param ctrl Host controller data structure
  350. * @return none
  351. */
  352. void xhci_acknowledge_event(struct xhci_ctrl *ctrl)
  353. {
  354. /* Advance our dequeue pointer to the next event */
  355. inc_deq(ctrl, ctrl->event_ring);
  356. /* Inform the hardware */
  357. xhci_writeq(&ctrl->ir_set->erst_dequeue,
  358. (uintptr_t)ctrl->event_ring->dequeue | ERST_EHB);
  359. }
  360. /**
  361. * Checks if there is a new event to handle on the event ring.
  362. *
  363. * @param ctrl Host controller data structure
  364. * @return 0 if failure else 1 on success
  365. */
  366. static int event_ready(struct xhci_ctrl *ctrl)
  367. {
  368. union xhci_trb *event;
  369. xhci_inval_cache((uintptr_t)ctrl->event_ring->dequeue,
  370. sizeof(union xhci_trb));
  371. event = ctrl->event_ring->dequeue;
  372. /* Does the HC or OS own the TRB? */
  373. if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
  374. ctrl->event_ring->cycle_state)
  375. return 0;
  376. return 1;
  377. }
  378. /**
  379. * Waits for a specific type of event and returns it. Discards unexpected
  380. * events. Caller *must* call xhci_acknowledge_event() after it is finished
  381. * processing the event, and must not access the returned pointer afterwards.
  382. *
  383. * @param ctrl Host controller data structure
  384. * @param expected TRB type expected from Event TRB
  385. * @return pointer to event trb
  386. */
  387. union xhci_trb *xhci_wait_for_event(struct xhci_ctrl *ctrl, trb_type expected)
  388. {
  389. trb_type type;
  390. unsigned long ts = get_timer(0);
  391. do {
  392. union xhci_trb *event = ctrl->event_ring->dequeue;
  393. if (!event_ready(ctrl))
  394. continue;
  395. type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags));
  396. if (type == expected)
  397. return event;
  398. if (type == TRB_PORT_STATUS)
  399. /* TODO: remove this once enumeration has been reworked */
  400. /*
  401. * Port status change events always have a
  402. * successful completion code
  403. */
  404. BUG_ON(GET_COMP_CODE(
  405. le32_to_cpu(event->generic.field[2])) !=
  406. COMP_SUCCESS);
  407. else
  408. printf("Unexpected XHCI event TRB, skipping... "
  409. "(%08x %08x %08x %08x)\n",
  410. le32_to_cpu(event->generic.field[0]),
  411. le32_to_cpu(event->generic.field[1]),
  412. le32_to_cpu(event->generic.field[2]),
  413. le32_to_cpu(event->generic.field[3]));
  414. xhci_acknowledge_event(ctrl);
  415. } while (get_timer(ts) < XHCI_TIMEOUT);
  416. if (expected == TRB_TRANSFER)
  417. return NULL;
  418. printf("XHCI timeout on event type %d... cannot recover.\n", expected);
  419. BUG();
  420. }
  421. /*
  422. * Stops transfer processing for an endpoint and throws away all unprocessed
  423. * TRBs by setting the xHC's dequeue pointer to our enqueue pointer. The next
  424. * xhci_bulk_tx/xhci_ctrl_tx on this enpoint will add new transfers there and
  425. * ring the doorbell, causing this endpoint to start working again.
  426. * (Careful: This will BUG() when there was no transfer in progress. Shouldn't
  427. * happen in practice for current uses and is too complicated to fix right now.)
  428. */
  429. static void abort_td(struct usb_device *udev, int ep_index)
  430. {
  431. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  432. struct xhci_ring *ring = ctrl->devs[udev->slot_id]->eps[ep_index].ring;
  433. union xhci_trb *event;
  434. u32 field;
  435. xhci_queue_command(ctrl, NULL, udev->slot_id, ep_index, TRB_STOP_RING);
  436. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  437. field = le32_to_cpu(event->trans_event.flags);
  438. BUG_ON(TRB_TO_SLOT_ID(field) != udev->slot_id);
  439. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  440. BUG_ON(GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len
  441. != COMP_STOP)));
  442. xhci_acknowledge_event(ctrl);
  443. event = xhci_wait_for_event(ctrl, TRB_COMPLETION);
  444. BUG_ON(TRB_TO_SLOT_ID(le32_to_cpu(event->event_cmd.flags))
  445. != udev->slot_id || GET_COMP_CODE(le32_to_cpu(
  446. event->event_cmd.status)) != COMP_SUCCESS);
  447. xhci_acknowledge_event(ctrl);
  448. xhci_queue_command(ctrl, (void *)((uintptr_t)ring->enqueue |
  449. ring->cycle_state), udev->slot_id, ep_index, TRB_SET_DEQ);
  450. event = xhci_wait_for_event(ctrl, TRB_COMPLETION);
  451. BUG_ON(TRB_TO_SLOT_ID(le32_to_cpu(event->event_cmd.flags))
  452. != udev->slot_id || GET_COMP_CODE(le32_to_cpu(
  453. event->event_cmd.status)) != COMP_SUCCESS);
  454. xhci_acknowledge_event(ctrl);
  455. }
  456. static void record_transfer_result(struct usb_device *udev,
  457. union xhci_trb *event, int length)
  458. {
  459. udev->act_len = min(length, length -
  460. (int)EVENT_TRB_LEN(le32_to_cpu(event->trans_event.transfer_len)));
  461. switch (GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len))) {
  462. case COMP_SUCCESS:
  463. BUG_ON(udev->act_len != length);
  464. /* fallthrough */
  465. case COMP_SHORT_TX:
  466. udev->status = 0;
  467. break;
  468. case COMP_STALL:
  469. udev->status = USB_ST_STALLED;
  470. break;
  471. case COMP_DB_ERR:
  472. case COMP_TRB_ERR:
  473. udev->status = USB_ST_BUF_ERR;
  474. break;
  475. case COMP_BABBLE:
  476. udev->status = USB_ST_BABBLE_DET;
  477. break;
  478. default:
  479. udev->status = 0x80; /* USB_ST_TOO_LAZY_TO_MAKE_A_NEW_MACRO */
  480. }
  481. }
  482. /**** Bulk and Control transfer methods ****/
  483. /**
  484. * Queues up the BULK Request
  485. *
  486. * @param udev pointer to the USB device structure
  487. * @param pipe contains the DIR_IN or OUT , devnum
  488. * @param length length of the buffer
  489. * @param buffer buffer to be read/written based on the request
  490. * @return returns 0 if successful else -1 on failure
  491. */
  492. int xhci_bulk_tx(struct usb_device *udev, unsigned long pipe,
  493. int length, void *buffer)
  494. {
  495. int num_trbs = 0;
  496. struct xhci_generic_trb *start_trb;
  497. bool first_trb = false;
  498. int start_cycle;
  499. u32 field = 0;
  500. u32 length_field = 0;
  501. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  502. int slot_id = udev->slot_id;
  503. int ep_index;
  504. struct xhci_virt_device *virt_dev;
  505. struct xhci_ep_ctx *ep_ctx;
  506. struct xhci_ring *ring; /* EP transfer ring */
  507. union xhci_trb *event;
  508. int running_total, trb_buff_len;
  509. unsigned int total_packet_count;
  510. int maxpacketsize;
  511. u64 addr;
  512. int ret;
  513. u32 trb_fields[4];
  514. u64 val_64 = (uintptr_t)buffer;
  515. debug("dev=%p, pipe=%lx, buffer=%p, length=%d\n",
  516. udev, pipe, buffer, length);
  517. ep_index = usb_pipe_ep_index(pipe);
  518. virt_dev = ctrl->devs[slot_id];
  519. xhci_inval_cache((uintptr_t)virt_dev->out_ctx->bytes,
  520. virt_dev->out_ctx->size);
  521. ep_ctx = xhci_get_ep_ctx(ctrl, virt_dev->out_ctx, ep_index);
  522. ring = virt_dev->eps[ep_index].ring;
  523. /*
  524. * How much data is (potentially) left before the 64KB boundary?
  525. * XHCI Spec puts restriction( TABLE 49 and 6.4.1 section of XHCI Spec)
  526. * that the buffer should not span 64KB boundary. if so
  527. * we send request in more than 1 TRB by chaining them.
  528. */
  529. running_total = TRB_MAX_BUFF_SIZE -
  530. (lower_32_bits(val_64) & (TRB_MAX_BUFF_SIZE - 1));
  531. trb_buff_len = running_total;
  532. running_total &= TRB_MAX_BUFF_SIZE - 1;
  533. /*
  534. * If there's some data on this 64KB chunk, or we have to send a
  535. * zero-length transfer, we need at least one TRB
  536. */
  537. if (running_total != 0 || length == 0)
  538. num_trbs++;
  539. /* How many more 64KB chunks to transfer, how many more TRBs? */
  540. while (running_total < length) {
  541. num_trbs++;
  542. running_total += TRB_MAX_BUFF_SIZE;
  543. }
  544. /*
  545. * XXX: Calling routine prepare_ring() called in place of
  546. * prepare_trasfer() as there in 'Linux' since we are not
  547. * maintaining multiple TDs/transfer at the same time.
  548. */
  549. ret = prepare_ring(ctrl, ring,
  550. le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK);
  551. if (ret < 0)
  552. return ret;
  553. /*
  554. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  555. * until we've finished creating all the other TRBs. The ring's cycle
  556. * state may change as we enqueue the other TRBs, so save it too.
  557. */
  558. start_trb = &ring->enqueue->generic;
  559. start_cycle = ring->cycle_state;
  560. running_total = 0;
  561. maxpacketsize = usb_maxpacket(udev, pipe);
  562. total_packet_count = DIV_ROUND_UP(length, maxpacketsize);
  563. /* How much data is in the first TRB? */
  564. /*
  565. * How much data is (potentially) left before the 64KB boundary?
  566. * XHCI Spec puts restriction( TABLE 49 and 6.4.1 section of XHCI Spec)
  567. * that the buffer should not span 64KB boundary. if so
  568. * we send request in more than 1 TRB by chaining them.
  569. */
  570. addr = val_64;
  571. if (trb_buff_len > length)
  572. trb_buff_len = length;
  573. first_trb = true;
  574. /* flush the buffer before use */
  575. xhci_flush_cache((uintptr_t)buffer, length);
  576. /* Queue the first TRB, even if it's zero-length */
  577. do {
  578. u32 remainder = 0;
  579. field = 0;
  580. /* Don't change the cycle bit of the first TRB until later */
  581. if (first_trb) {
  582. first_trb = false;
  583. if (start_cycle == 0)
  584. field |= TRB_CYCLE;
  585. } else {
  586. field |= ring->cycle_state;
  587. }
  588. /*
  589. * Chain all the TRBs together; clear the chain bit in the last
  590. * TRB to indicate it's the last TRB in the chain.
  591. */
  592. if (num_trbs > 1)
  593. field |= TRB_CHAIN;
  594. else
  595. field |= TRB_IOC;
  596. /* Only set interrupt on short packet for IN endpoints */
  597. if (usb_pipein(pipe))
  598. field |= TRB_ISP;
  599. /* Set the TRB length, TD size, and interrupter fields. */
  600. if (HC_VERSION(xhci_readl(&ctrl->hccr->cr_capbase)) < 0x100)
  601. remainder = xhci_td_remainder(length - running_total);
  602. else
  603. remainder = xhci_v1_0_td_remainder(running_total,
  604. trb_buff_len,
  605. total_packet_count,
  606. maxpacketsize,
  607. num_trbs - 1);
  608. length_field = ((trb_buff_len & TRB_LEN_MASK) |
  609. remainder |
  610. ((0 & TRB_INTR_TARGET_MASK) <<
  611. TRB_INTR_TARGET_SHIFT));
  612. trb_fields[0] = lower_32_bits(addr);
  613. trb_fields[1] = upper_32_bits(addr);
  614. trb_fields[2] = length_field;
  615. trb_fields[3] = field | (TRB_NORMAL << TRB_TYPE_SHIFT);
  616. queue_trb(ctrl, ring, (num_trbs > 1), trb_fields);
  617. --num_trbs;
  618. running_total += trb_buff_len;
  619. /* Calculate length for next transfer */
  620. addr += trb_buff_len;
  621. trb_buff_len = min((length - running_total), TRB_MAX_BUFF_SIZE);
  622. } while (running_total < length);
  623. giveback_first_trb(udev, ep_index, start_cycle, start_trb);
  624. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  625. if (!event) {
  626. debug("XHCI bulk transfer timed out, aborting...\n");
  627. abort_td(udev, ep_index);
  628. udev->status = USB_ST_NAK_REC; /* closest thing to a timeout */
  629. udev->act_len = 0;
  630. return -ETIMEDOUT;
  631. }
  632. field = le32_to_cpu(event->trans_event.flags);
  633. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  634. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  635. BUG_ON(*(void **)(uintptr_t)le64_to_cpu(event->trans_event.buffer) -
  636. buffer > (size_t)length);
  637. record_transfer_result(udev, event, length);
  638. xhci_acknowledge_event(ctrl);
  639. xhci_inval_cache((uintptr_t)buffer, length);
  640. return (udev->status != USB_ST_NOT_PROC) ? 0 : -1;
  641. }
  642. /**
  643. * Queues up the Control Transfer Request
  644. *
  645. * @param udev pointer to the USB device structure
  646. * @param pipe contains the DIR_IN or OUT , devnum
  647. * @param req request type
  648. * @param length length of the buffer
  649. * @param buffer buffer to be read/written based on the request
  650. * @return returns 0 if successful else error code on failure
  651. */
  652. int xhci_ctrl_tx(struct usb_device *udev, unsigned long pipe,
  653. struct devrequest *req, int length,
  654. void *buffer)
  655. {
  656. int ret;
  657. int start_cycle;
  658. int num_trbs;
  659. u32 field;
  660. u32 length_field;
  661. u64 buf_64 = 0;
  662. struct xhci_generic_trb *start_trb;
  663. struct xhci_ctrl *ctrl = xhci_get_ctrl(udev);
  664. int slot_id = udev->slot_id;
  665. int ep_index;
  666. u32 trb_fields[4];
  667. struct xhci_virt_device *virt_dev = ctrl->devs[slot_id];
  668. struct xhci_ring *ep_ring;
  669. union xhci_trb *event;
  670. debug("req=%u (%#x), type=%u (%#x), value=%u (%#x), index=%u\n",
  671. req->request, req->request,
  672. req->requesttype, req->requesttype,
  673. le16_to_cpu(req->value), le16_to_cpu(req->value),
  674. le16_to_cpu(req->index));
  675. ep_index = usb_pipe_ep_index(pipe);
  676. ep_ring = virt_dev->eps[ep_index].ring;
  677. /*
  678. * Check to see if the max packet size for the default control
  679. * endpoint changed during FS device enumeration
  680. */
  681. if (udev->speed == USB_SPEED_FULL) {
  682. ret = xhci_check_maxpacket(udev);
  683. if (ret < 0)
  684. return ret;
  685. }
  686. xhci_inval_cache((uintptr_t)virt_dev->out_ctx->bytes,
  687. virt_dev->out_ctx->size);
  688. struct xhci_ep_ctx *ep_ctx = NULL;
  689. ep_ctx = xhci_get_ep_ctx(ctrl, virt_dev->out_ctx, ep_index);
  690. /* 1 TRB for setup, 1 for status */
  691. num_trbs = 2;
  692. /*
  693. * Don't need to check if we need additional event data and normal TRBs,
  694. * since data in control transfers will never get bigger than 16MB
  695. * XXX: can we get a buffer that crosses 64KB boundaries?
  696. */
  697. if (length > 0)
  698. num_trbs++;
  699. /*
  700. * XXX: Calling routine prepare_ring() called in place of
  701. * prepare_trasfer() as there in 'Linux' since we are not
  702. * maintaining multiple TDs/transfer at the same time.
  703. */
  704. ret = prepare_ring(ctrl, ep_ring,
  705. le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK);
  706. if (ret < 0)
  707. return ret;
  708. /*
  709. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  710. * until we've finished creating all the other TRBs. The ring's cycle
  711. * state may change as we enqueue the other TRBs, so save it too.
  712. */
  713. start_trb = &ep_ring->enqueue->generic;
  714. start_cycle = ep_ring->cycle_state;
  715. debug("start_trb %p, start_cycle %d\n", start_trb, start_cycle);
  716. /* Queue setup TRB - see section 6.4.1.2.1 */
  717. /* FIXME better way to translate setup_packet into two u32 fields? */
  718. field = 0;
  719. field |= TRB_IDT | (TRB_SETUP << TRB_TYPE_SHIFT);
  720. if (start_cycle == 0)
  721. field |= 0x1;
  722. /* xHCI 1.0 6.4.1.2.1: Transfer Type field */
  723. if (HC_VERSION(xhci_readl(&ctrl->hccr->cr_capbase)) >= 0x100) {
  724. if (length > 0) {
  725. if (req->requesttype & USB_DIR_IN)
  726. field |= (TRB_DATA_IN << TRB_TX_TYPE_SHIFT);
  727. else
  728. field |= (TRB_DATA_OUT << TRB_TX_TYPE_SHIFT);
  729. }
  730. }
  731. debug("req->requesttype = %d, req->request = %d,"
  732. "le16_to_cpu(req->value) = %d,"
  733. "le16_to_cpu(req->index) = %d,"
  734. "le16_to_cpu(req->length) = %d\n",
  735. req->requesttype, req->request, le16_to_cpu(req->value),
  736. le16_to_cpu(req->index), le16_to_cpu(req->length));
  737. trb_fields[0] = req->requesttype | req->request << 8 |
  738. le16_to_cpu(req->value) << 16;
  739. trb_fields[1] = le16_to_cpu(req->index) |
  740. le16_to_cpu(req->length) << 16;
  741. /* TRB_LEN | (TRB_INTR_TARGET) */
  742. trb_fields[2] = (8 | ((0 & TRB_INTR_TARGET_MASK) <<
  743. TRB_INTR_TARGET_SHIFT));
  744. /* Immediate data in pointer */
  745. trb_fields[3] = field;
  746. queue_trb(ctrl, ep_ring, true, trb_fields);
  747. /* Re-initializing field to zero */
  748. field = 0;
  749. /* If there's data, queue data TRBs */
  750. /* Only set interrupt on short packet for IN endpoints */
  751. if (usb_pipein(pipe))
  752. field = TRB_ISP | (TRB_DATA << TRB_TYPE_SHIFT);
  753. else
  754. field = (TRB_DATA << TRB_TYPE_SHIFT);
  755. length_field = (length & TRB_LEN_MASK) | xhci_td_remainder(length) |
  756. ((0 & TRB_INTR_TARGET_MASK) << TRB_INTR_TARGET_SHIFT);
  757. debug("length_field = %d, length = %d,"
  758. "xhci_td_remainder(length) = %d , TRB_INTR_TARGET(0) = %d\n",
  759. length_field, (length & TRB_LEN_MASK),
  760. xhci_td_remainder(length), 0);
  761. if (length > 0) {
  762. if (req->requesttype & USB_DIR_IN)
  763. field |= TRB_DIR_IN;
  764. buf_64 = (uintptr_t)buffer;
  765. trb_fields[0] = lower_32_bits(buf_64);
  766. trb_fields[1] = upper_32_bits(buf_64);
  767. trb_fields[2] = length_field;
  768. trb_fields[3] = field | ep_ring->cycle_state;
  769. xhci_flush_cache((uintptr_t)buffer, length);
  770. queue_trb(ctrl, ep_ring, true, trb_fields);
  771. }
  772. /*
  773. * Queue status TRB -
  774. * see Table 7 and sections 4.11.2.2 and 6.4.1.2.3
  775. */
  776. /* If the device sent data, the status stage is an OUT transfer */
  777. field = 0;
  778. if (length > 0 && req->requesttype & USB_DIR_IN)
  779. field = 0;
  780. else
  781. field = TRB_DIR_IN;
  782. trb_fields[0] = 0;
  783. trb_fields[1] = 0;
  784. trb_fields[2] = ((0 & TRB_INTR_TARGET_MASK) << TRB_INTR_TARGET_SHIFT);
  785. /* Event on completion */
  786. trb_fields[3] = field | TRB_IOC |
  787. (TRB_STATUS << TRB_TYPE_SHIFT) |
  788. ep_ring->cycle_state;
  789. queue_trb(ctrl, ep_ring, false, trb_fields);
  790. giveback_first_trb(udev, ep_index, start_cycle, start_trb);
  791. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  792. if (!event)
  793. goto abort;
  794. field = le32_to_cpu(event->trans_event.flags);
  795. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  796. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  797. record_transfer_result(udev, event, length);
  798. xhci_acknowledge_event(ctrl);
  799. /* Invalidate buffer to make it available to usb-core */
  800. if (length > 0)
  801. xhci_inval_cache((uintptr_t)buffer, length);
  802. if (GET_COMP_CODE(le32_to_cpu(event->trans_event.transfer_len))
  803. == COMP_SHORT_TX) {
  804. /* Short data stage, clear up additional status stage event */
  805. event = xhci_wait_for_event(ctrl, TRB_TRANSFER);
  806. if (!event)
  807. goto abort;
  808. BUG_ON(TRB_TO_SLOT_ID(field) != slot_id);
  809. BUG_ON(TRB_TO_EP_INDEX(field) != ep_index);
  810. xhci_acknowledge_event(ctrl);
  811. }
  812. return (udev->status != USB_ST_NOT_PROC) ? 0 : -1;
  813. abort:
  814. debug("XHCI control transfer timed out, aborting...\n");
  815. abort_td(udev, ep_index);
  816. udev->status = USB_ST_NAK_REC;
  817. udev->act_len = 0;
  818. return -ETIMEDOUT;
  819. }