libahci.c 67 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * libahci.c - Common AHCI SATA low-level routines
  4. *
  5. * Maintained by: Tejun Heo <tj@kernel.org>
  6. * Please ALWAYS copy linux-ide@vger.kernel.org
  7. * on emails.
  8. *
  9. * Copyright 2004-2005 Red Hat, Inc.
  10. *
  11. * libata documentation is available via 'make {ps|pdf}docs',
  12. * as Documentation/driver-api/libata.rst
  13. *
  14. * AHCI hardware documentation:
  15. * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  16. * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  17. */
  18. #include <linux/kernel.h>
  19. #include <linux/gfp.h>
  20. #include <linux/module.h>
  21. #include <linux/nospec.h>
  22. #include <linux/blkdev.h>
  23. #include <linux/delay.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/dma-mapping.h>
  26. #include <linux/device.h>
  27. #include <scsi/scsi_host.h>
  28. #include <scsi/scsi_cmnd.h>
  29. #include <linux/libata.h>
  30. #include <linux/pci.h>
  31. #include "ahci.h"
  32. #include "libata.h"
  33. static int ahci_skip_host_reset;
  34. int ahci_ignore_sss;
  35. EXPORT_SYMBOL_GPL(ahci_ignore_sss);
  36. module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
  37. MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
  38. module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
  39. MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
  40. static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  41. unsigned hints);
  42. static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
  43. static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  44. size_t size);
  45. static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  46. ssize_t size);
  47. static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
  48. static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
  49. static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
  50. static int ahci_port_start(struct ata_port *ap);
  51. static void ahci_port_stop(struct ata_port *ap);
  52. static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc);
  53. static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
  54. static void ahci_freeze(struct ata_port *ap);
  55. static void ahci_thaw(struct ata_port *ap);
  56. static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep);
  57. static void ahci_enable_fbs(struct ata_port *ap);
  58. static void ahci_disable_fbs(struct ata_port *ap);
  59. static void ahci_pmp_attach(struct ata_port *ap);
  60. static void ahci_pmp_detach(struct ata_port *ap);
  61. static int ahci_softreset(struct ata_link *link, unsigned int *class,
  62. unsigned long deadline);
  63. static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  64. unsigned long deadline);
  65. static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  66. unsigned long deadline);
  67. static void ahci_postreset(struct ata_link *link, unsigned int *class);
  68. static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
  69. static void ahci_dev_config(struct ata_device *dev);
  70. #ifdef CONFIG_PM
  71. static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
  72. #endif
  73. static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
  74. static ssize_t ahci_activity_store(struct ata_device *dev,
  75. enum sw_activity val);
  76. static void ahci_init_sw_activity(struct ata_link *link);
  77. static ssize_t ahci_show_host_caps(struct device *dev,
  78. struct device_attribute *attr, char *buf);
  79. static ssize_t ahci_show_host_cap2(struct device *dev,
  80. struct device_attribute *attr, char *buf);
  81. static ssize_t ahci_show_host_version(struct device *dev,
  82. struct device_attribute *attr, char *buf);
  83. static ssize_t ahci_show_port_cmd(struct device *dev,
  84. struct device_attribute *attr, char *buf);
  85. static ssize_t ahci_read_em_buffer(struct device *dev,
  86. struct device_attribute *attr, char *buf);
  87. static ssize_t ahci_store_em_buffer(struct device *dev,
  88. struct device_attribute *attr,
  89. const char *buf, size_t size);
  90. static ssize_t ahci_show_em_supported(struct device *dev,
  91. struct device_attribute *attr, char *buf);
  92. static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance);
  93. static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
  94. static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
  95. static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
  96. static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
  97. static DEVICE_ATTR(em_buffer, S_IWUSR | S_IRUGO,
  98. ahci_read_em_buffer, ahci_store_em_buffer);
  99. static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
  100. struct device_attribute *ahci_shost_attrs[] = {
  101. &dev_attr_link_power_management_policy,
  102. &dev_attr_em_message_type,
  103. &dev_attr_em_message,
  104. &dev_attr_ahci_host_caps,
  105. &dev_attr_ahci_host_cap2,
  106. &dev_attr_ahci_host_version,
  107. &dev_attr_ahci_port_cmd,
  108. &dev_attr_em_buffer,
  109. &dev_attr_em_message_supported,
  110. NULL
  111. };
  112. EXPORT_SYMBOL_GPL(ahci_shost_attrs);
  113. struct device_attribute *ahci_sdev_attrs[] = {
  114. &dev_attr_sw_activity,
  115. &dev_attr_unload_heads,
  116. &dev_attr_ncq_prio_enable,
  117. NULL
  118. };
  119. EXPORT_SYMBOL_GPL(ahci_sdev_attrs);
  120. struct ata_port_operations ahci_ops = {
  121. .inherits = &sata_pmp_port_ops,
  122. .qc_defer = ahci_pmp_qc_defer,
  123. .qc_prep = ahci_qc_prep,
  124. .qc_issue = ahci_qc_issue,
  125. .qc_fill_rtf = ahci_qc_fill_rtf,
  126. .freeze = ahci_freeze,
  127. .thaw = ahci_thaw,
  128. .softreset = ahci_softreset,
  129. .hardreset = ahci_hardreset,
  130. .postreset = ahci_postreset,
  131. .pmp_softreset = ahci_softreset,
  132. .error_handler = ahci_error_handler,
  133. .post_internal_cmd = ahci_post_internal_cmd,
  134. .dev_config = ahci_dev_config,
  135. .scr_read = ahci_scr_read,
  136. .scr_write = ahci_scr_write,
  137. .pmp_attach = ahci_pmp_attach,
  138. .pmp_detach = ahci_pmp_detach,
  139. .set_lpm = ahci_set_lpm,
  140. .em_show = ahci_led_show,
  141. .em_store = ahci_led_store,
  142. .sw_activity_show = ahci_activity_show,
  143. .sw_activity_store = ahci_activity_store,
  144. .transmit_led_message = ahci_transmit_led_message,
  145. #ifdef CONFIG_PM
  146. .port_suspend = ahci_port_suspend,
  147. .port_resume = ahci_port_resume,
  148. #endif
  149. .port_start = ahci_port_start,
  150. .port_stop = ahci_port_stop,
  151. };
  152. EXPORT_SYMBOL_GPL(ahci_ops);
  153. struct ata_port_operations ahci_pmp_retry_srst_ops = {
  154. .inherits = &ahci_ops,
  155. .softreset = ahci_pmp_retry_softreset,
  156. };
  157. EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
  158. static bool ahci_em_messages __read_mostly = true;
  159. module_param(ahci_em_messages, bool, 0444);
  160. /* add other LED protocol types when they become supported */
  161. MODULE_PARM_DESC(ahci_em_messages,
  162. "AHCI Enclosure Management Message control (0 = off, 1 = on)");
  163. /* device sleep idle timeout in ms */
  164. static int devslp_idle_timeout __read_mostly = 1000;
  165. module_param(devslp_idle_timeout, int, 0644);
  166. MODULE_PARM_DESC(devslp_idle_timeout, "device sleep idle timeout");
  167. static void ahci_enable_ahci(void __iomem *mmio)
  168. {
  169. int i;
  170. u32 tmp;
  171. /* turn on AHCI_EN */
  172. tmp = readl(mmio + HOST_CTL);
  173. if (tmp & HOST_AHCI_EN)
  174. return;
  175. /* Some controllers need AHCI_EN to be written multiple times.
  176. * Try a few times before giving up.
  177. */
  178. for (i = 0; i < 5; i++) {
  179. tmp |= HOST_AHCI_EN;
  180. writel(tmp, mmio + HOST_CTL);
  181. tmp = readl(mmio + HOST_CTL); /* flush && sanity check */
  182. if (tmp & HOST_AHCI_EN)
  183. return;
  184. msleep(10);
  185. }
  186. WARN_ON(1);
  187. }
  188. /**
  189. * ahci_rpm_get_port - Make sure the port is powered on
  190. * @ap: Port to power on
  191. *
  192. * Whenever there is need to access the AHCI host registers outside of
  193. * normal execution paths, call this function to make sure the host is
  194. * actually powered on.
  195. */
  196. static int ahci_rpm_get_port(struct ata_port *ap)
  197. {
  198. return pm_runtime_get_sync(ap->dev);
  199. }
  200. /**
  201. * ahci_rpm_put_port - Undoes ahci_rpm_get_port()
  202. * @ap: Port to power down
  203. *
  204. * Undoes ahci_rpm_get_port() and possibly powers down the AHCI host
  205. * if it has no more active users.
  206. */
  207. static void ahci_rpm_put_port(struct ata_port *ap)
  208. {
  209. pm_runtime_put(ap->dev);
  210. }
  211. static ssize_t ahci_show_host_caps(struct device *dev,
  212. struct device_attribute *attr, char *buf)
  213. {
  214. struct Scsi_Host *shost = class_to_shost(dev);
  215. struct ata_port *ap = ata_shost_to_port(shost);
  216. struct ahci_host_priv *hpriv = ap->host->private_data;
  217. return sprintf(buf, "%x\n", hpriv->cap);
  218. }
  219. static ssize_t ahci_show_host_cap2(struct device *dev,
  220. struct device_attribute *attr, char *buf)
  221. {
  222. struct Scsi_Host *shost = class_to_shost(dev);
  223. struct ata_port *ap = ata_shost_to_port(shost);
  224. struct ahci_host_priv *hpriv = ap->host->private_data;
  225. return sprintf(buf, "%x\n", hpriv->cap2);
  226. }
  227. static ssize_t ahci_show_host_version(struct device *dev,
  228. struct device_attribute *attr, char *buf)
  229. {
  230. struct Scsi_Host *shost = class_to_shost(dev);
  231. struct ata_port *ap = ata_shost_to_port(shost);
  232. struct ahci_host_priv *hpriv = ap->host->private_data;
  233. return sprintf(buf, "%x\n", hpriv->version);
  234. }
  235. static ssize_t ahci_show_port_cmd(struct device *dev,
  236. struct device_attribute *attr, char *buf)
  237. {
  238. struct Scsi_Host *shost = class_to_shost(dev);
  239. struct ata_port *ap = ata_shost_to_port(shost);
  240. void __iomem *port_mmio = ahci_port_base(ap);
  241. ssize_t ret;
  242. ahci_rpm_get_port(ap);
  243. ret = sprintf(buf, "%x\n", readl(port_mmio + PORT_CMD));
  244. ahci_rpm_put_port(ap);
  245. return ret;
  246. }
  247. static ssize_t ahci_read_em_buffer(struct device *dev,
  248. struct device_attribute *attr, char *buf)
  249. {
  250. struct Scsi_Host *shost = class_to_shost(dev);
  251. struct ata_port *ap = ata_shost_to_port(shost);
  252. struct ahci_host_priv *hpriv = ap->host->private_data;
  253. void __iomem *mmio = hpriv->mmio;
  254. void __iomem *em_mmio = mmio + hpriv->em_loc;
  255. u32 em_ctl, msg;
  256. unsigned long flags;
  257. size_t count;
  258. int i;
  259. ahci_rpm_get_port(ap);
  260. spin_lock_irqsave(ap->lock, flags);
  261. em_ctl = readl(mmio + HOST_EM_CTL);
  262. if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||
  263. !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
  264. spin_unlock_irqrestore(ap->lock, flags);
  265. ahci_rpm_put_port(ap);
  266. return -EINVAL;
  267. }
  268. if (!(em_ctl & EM_CTL_MR)) {
  269. spin_unlock_irqrestore(ap->lock, flags);
  270. ahci_rpm_put_port(ap);
  271. return -EAGAIN;
  272. }
  273. if (!(em_ctl & EM_CTL_SMB))
  274. em_mmio += hpriv->em_buf_sz;
  275. count = hpriv->em_buf_sz;
  276. /* the count should not be larger than PAGE_SIZE */
  277. if (count > PAGE_SIZE) {
  278. if (printk_ratelimit())
  279. ata_port_warn(ap,
  280. "EM read buffer size too large: "
  281. "buffer size %u, page size %lu\n",
  282. hpriv->em_buf_sz, PAGE_SIZE);
  283. count = PAGE_SIZE;
  284. }
  285. for (i = 0; i < count; i += 4) {
  286. msg = readl(em_mmio + i);
  287. buf[i] = msg & 0xff;
  288. buf[i + 1] = (msg >> 8) & 0xff;
  289. buf[i + 2] = (msg >> 16) & 0xff;
  290. buf[i + 3] = (msg >> 24) & 0xff;
  291. }
  292. spin_unlock_irqrestore(ap->lock, flags);
  293. ahci_rpm_put_port(ap);
  294. return i;
  295. }
  296. static ssize_t ahci_store_em_buffer(struct device *dev,
  297. struct device_attribute *attr,
  298. const char *buf, size_t size)
  299. {
  300. struct Scsi_Host *shost = class_to_shost(dev);
  301. struct ata_port *ap = ata_shost_to_port(shost);
  302. struct ahci_host_priv *hpriv = ap->host->private_data;
  303. void __iomem *mmio = hpriv->mmio;
  304. void __iomem *em_mmio = mmio + hpriv->em_loc;
  305. const unsigned char *msg_buf = buf;
  306. u32 em_ctl, msg;
  307. unsigned long flags;
  308. int i;
  309. /* check size validity */
  310. if (!(ap->flags & ATA_FLAG_EM) ||
  311. !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||
  312. size % 4 || size > hpriv->em_buf_sz)
  313. return -EINVAL;
  314. ahci_rpm_get_port(ap);
  315. spin_lock_irqsave(ap->lock, flags);
  316. em_ctl = readl(mmio + HOST_EM_CTL);
  317. if (em_ctl & EM_CTL_TM) {
  318. spin_unlock_irqrestore(ap->lock, flags);
  319. ahci_rpm_put_port(ap);
  320. return -EBUSY;
  321. }
  322. for (i = 0; i < size; i += 4) {
  323. msg = msg_buf[i] | msg_buf[i + 1] << 8 |
  324. msg_buf[i + 2] << 16 | msg_buf[i + 3] << 24;
  325. writel(msg, em_mmio + i);
  326. }
  327. writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
  328. spin_unlock_irqrestore(ap->lock, flags);
  329. ahci_rpm_put_port(ap);
  330. return size;
  331. }
  332. static ssize_t ahci_show_em_supported(struct device *dev,
  333. struct device_attribute *attr, char *buf)
  334. {
  335. struct Scsi_Host *shost = class_to_shost(dev);
  336. struct ata_port *ap = ata_shost_to_port(shost);
  337. struct ahci_host_priv *hpriv = ap->host->private_data;
  338. void __iomem *mmio = hpriv->mmio;
  339. u32 em_ctl;
  340. ahci_rpm_get_port(ap);
  341. em_ctl = readl(mmio + HOST_EM_CTL);
  342. ahci_rpm_put_port(ap);
  343. return sprintf(buf, "%s%s%s%s\n",
  344. em_ctl & EM_CTL_LED ? "led " : "",
  345. em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
  346. em_ctl & EM_CTL_SES ? "ses-2 " : "",
  347. em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
  348. }
  349. /**
  350. * ahci_save_initial_config - Save and fixup initial config values
  351. * @dev: target AHCI device
  352. * @hpriv: host private area to store config values
  353. *
  354. * Some registers containing configuration info might be setup by
  355. * BIOS and might be cleared on reset. This function saves the
  356. * initial values of those registers into @hpriv such that they
  357. * can be restored after controller reset.
  358. *
  359. * If inconsistent, config values are fixed up by this function.
  360. *
  361. * If it is not set already this function sets hpriv->start_engine to
  362. * ahci_start_engine.
  363. *
  364. * LOCKING:
  365. * None.
  366. */
  367. void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv)
  368. {
  369. void __iomem *mmio = hpriv->mmio;
  370. u32 cap, cap2, vers, port_map;
  371. int i;
  372. /* make sure AHCI mode is enabled before accessing CAP */
  373. ahci_enable_ahci(mmio);
  374. /* Values prefixed with saved_ are written back to host after
  375. * reset. Values without are used for driver operation.
  376. */
  377. hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
  378. hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
  379. /* CAP2 register is only defined for AHCI 1.2 and later */
  380. vers = readl(mmio + HOST_VERSION);
  381. if ((vers >> 16) > 1 ||
  382. ((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
  383. hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);
  384. else
  385. hpriv->saved_cap2 = cap2 = 0;
  386. /* some chips have errata preventing 64bit use */
  387. if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
  388. dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
  389. cap &= ~HOST_CAP_64;
  390. }
  391. if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
  392. dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
  393. cap &= ~HOST_CAP_NCQ;
  394. }
  395. if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
  396. dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
  397. cap |= HOST_CAP_NCQ;
  398. }
  399. if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
  400. dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
  401. cap &= ~HOST_CAP_PMP;
  402. }
  403. if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
  404. dev_info(dev,
  405. "controller can't do SNTF, turning off CAP_SNTF\n");
  406. cap &= ~HOST_CAP_SNTF;
  407. }
  408. if ((cap2 & HOST_CAP2_SDS) && (hpriv->flags & AHCI_HFLAG_NO_DEVSLP)) {
  409. dev_info(dev,
  410. "controller can't do DEVSLP, turning off\n");
  411. cap2 &= ~HOST_CAP2_SDS;
  412. cap2 &= ~HOST_CAP2_SADM;
  413. }
  414. if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
  415. dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
  416. cap |= HOST_CAP_FBS;
  417. }
  418. if ((cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_NO_FBS)) {
  419. dev_info(dev, "controller can't do FBS, turning off CAP_FBS\n");
  420. cap &= ~HOST_CAP_FBS;
  421. }
  422. if (!(cap & HOST_CAP_ALPM) && (hpriv->flags & AHCI_HFLAG_YES_ALPM)) {
  423. dev_info(dev, "controller can do ALPM, turning on CAP_ALPM\n");
  424. cap |= HOST_CAP_ALPM;
  425. }
  426. if ((cap & HOST_CAP_SXS) && (hpriv->flags & AHCI_HFLAG_NO_SXS)) {
  427. dev_info(dev, "controller does not support SXS, disabling CAP_SXS\n");
  428. cap &= ~HOST_CAP_SXS;
  429. }
  430. if (hpriv->force_port_map && port_map != hpriv->force_port_map) {
  431. dev_info(dev, "forcing port_map 0x%x -> 0x%x\n",
  432. port_map, hpriv->force_port_map);
  433. port_map = hpriv->force_port_map;
  434. hpriv->saved_port_map = port_map;
  435. }
  436. if (hpriv->mask_port_map) {
  437. dev_warn(dev, "masking port_map 0x%x -> 0x%x\n",
  438. port_map,
  439. port_map & hpriv->mask_port_map);
  440. port_map &= hpriv->mask_port_map;
  441. }
  442. /* cross check port_map and cap.n_ports */
  443. if (port_map) {
  444. int map_ports = 0;
  445. for (i = 0; i < AHCI_MAX_PORTS; i++)
  446. if (port_map & (1 << i))
  447. map_ports++;
  448. /* If PI has more ports than n_ports, whine, clear
  449. * port_map and let it be generated from n_ports.
  450. */
  451. if (map_ports > ahci_nr_ports(cap)) {
  452. dev_warn(dev,
  453. "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
  454. port_map, ahci_nr_ports(cap));
  455. port_map = 0;
  456. }
  457. }
  458. /* fabricate port_map from cap.nr_ports for < AHCI 1.3 */
  459. if (!port_map && vers < 0x10300) {
  460. port_map = (1 << ahci_nr_ports(cap)) - 1;
  461. dev_warn(dev, "forcing PORTS_IMPL to 0x%x\n", port_map);
  462. /* write the fixed up value to the PI register */
  463. hpriv->saved_port_map = port_map;
  464. }
  465. /* record values to use during operation */
  466. hpriv->cap = cap;
  467. hpriv->cap2 = cap2;
  468. hpriv->version = readl(mmio + HOST_VERSION);
  469. hpriv->port_map = port_map;
  470. if (!hpriv->start_engine)
  471. hpriv->start_engine = ahci_start_engine;
  472. if (!hpriv->stop_engine)
  473. hpriv->stop_engine = ahci_stop_engine;
  474. if (!hpriv->irq_handler)
  475. hpriv->irq_handler = ahci_single_level_irq_intr;
  476. }
  477. EXPORT_SYMBOL_GPL(ahci_save_initial_config);
  478. /**
  479. * ahci_restore_initial_config - Restore initial config
  480. * @host: target ATA host
  481. *
  482. * Restore initial config stored by ahci_save_initial_config().
  483. *
  484. * LOCKING:
  485. * None.
  486. */
  487. static void ahci_restore_initial_config(struct ata_host *host)
  488. {
  489. struct ahci_host_priv *hpriv = host->private_data;
  490. void __iomem *mmio = hpriv->mmio;
  491. writel(hpriv->saved_cap, mmio + HOST_CAP);
  492. if (hpriv->saved_cap2)
  493. writel(hpriv->saved_cap2, mmio + HOST_CAP2);
  494. writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
  495. (void) readl(mmio + HOST_PORTS_IMPL); /* flush */
  496. }
  497. static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
  498. {
  499. static const int offset[] = {
  500. [SCR_STATUS] = PORT_SCR_STAT,
  501. [SCR_CONTROL] = PORT_SCR_CTL,
  502. [SCR_ERROR] = PORT_SCR_ERR,
  503. [SCR_ACTIVE] = PORT_SCR_ACT,
  504. [SCR_NOTIFICATION] = PORT_SCR_NTF,
  505. };
  506. struct ahci_host_priv *hpriv = ap->host->private_data;
  507. if (sc_reg < ARRAY_SIZE(offset) &&
  508. (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
  509. return offset[sc_reg];
  510. return 0;
  511. }
  512. static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
  513. {
  514. void __iomem *port_mmio = ahci_port_base(link->ap);
  515. int offset = ahci_scr_offset(link->ap, sc_reg);
  516. if (offset) {
  517. *val = readl(port_mmio + offset);
  518. return 0;
  519. }
  520. return -EINVAL;
  521. }
  522. static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
  523. {
  524. void __iomem *port_mmio = ahci_port_base(link->ap);
  525. int offset = ahci_scr_offset(link->ap, sc_reg);
  526. if (offset) {
  527. writel(val, port_mmio + offset);
  528. return 0;
  529. }
  530. return -EINVAL;
  531. }
  532. void ahci_start_engine(struct ata_port *ap)
  533. {
  534. void __iomem *port_mmio = ahci_port_base(ap);
  535. u32 tmp;
  536. /* start DMA */
  537. tmp = readl(port_mmio + PORT_CMD);
  538. tmp |= PORT_CMD_START;
  539. writel(tmp, port_mmio + PORT_CMD);
  540. readl(port_mmio + PORT_CMD); /* flush */
  541. }
  542. EXPORT_SYMBOL_GPL(ahci_start_engine);
  543. int ahci_stop_engine(struct ata_port *ap)
  544. {
  545. void __iomem *port_mmio = ahci_port_base(ap);
  546. struct ahci_host_priv *hpriv = ap->host->private_data;
  547. u32 tmp;
  548. /*
  549. * On some controllers, stopping a port's DMA engine while the port
  550. * is in ALPM state (partial or slumber) results in failures on
  551. * subsequent DMA engine starts. For those controllers, put the
  552. * port back in active state before stopping its DMA engine.
  553. */
  554. if ((hpriv->flags & AHCI_HFLAG_WAKE_BEFORE_STOP) &&
  555. (ap->link.lpm_policy > ATA_LPM_MAX_POWER) &&
  556. ahci_set_lpm(&ap->link, ATA_LPM_MAX_POWER, ATA_LPM_WAKE_ONLY)) {
  557. dev_err(ap->host->dev, "Failed to wake up port before engine stop\n");
  558. return -EIO;
  559. }
  560. tmp = readl(port_mmio + PORT_CMD);
  561. /* check if the HBA is idle */
  562. if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
  563. return 0;
  564. /*
  565. * Don't try to issue commands but return with ENODEV if the
  566. * AHCI controller not available anymore (e.g. due to PCIe hot
  567. * unplugging). Otherwise a 500ms delay for each port is added.
  568. */
  569. if (tmp == 0xffffffff) {
  570. dev_err(ap->host->dev, "AHCI controller unavailable!\n");
  571. return -ENODEV;
  572. }
  573. /* setting HBA to idle */
  574. tmp &= ~PORT_CMD_START;
  575. writel(tmp, port_mmio + PORT_CMD);
  576. /* wait for engine to stop. This could be as long as 500 msec */
  577. tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
  578. PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
  579. if (tmp & PORT_CMD_LIST_ON)
  580. return -EIO;
  581. return 0;
  582. }
  583. EXPORT_SYMBOL_GPL(ahci_stop_engine);
  584. void ahci_start_fis_rx(struct ata_port *ap)
  585. {
  586. void __iomem *port_mmio = ahci_port_base(ap);
  587. struct ahci_host_priv *hpriv = ap->host->private_data;
  588. struct ahci_port_priv *pp = ap->private_data;
  589. u32 tmp;
  590. /* set FIS registers */
  591. if (hpriv->cap & HOST_CAP_64)
  592. writel((pp->cmd_slot_dma >> 16) >> 16,
  593. port_mmio + PORT_LST_ADDR_HI);
  594. writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
  595. if (hpriv->cap & HOST_CAP_64)
  596. writel((pp->rx_fis_dma >> 16) >> 16,
  597. port_mmio + PORT_FIS_ADDR_HI);
  598. writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
  599. /* enable FIS reception */
  600. tmp = readl(port_mmio + PORT_CMD);
  601. tmp |= PORT_CMD_FIS_RX;
  602. writel(tmp, port_mmio + PORT_CMD);
  603. /* flush */
  604. readl(port_mmio + PORT_CMD);
  605. }
  606. EXPORT_SYMBOL_GPL(ahci_start_fis_rx);
  607. static int ahci_stop_fis_rx(struct ata_port *ap)
  608. {
  609. void __iomem *port_mmio = ahci_port_base(ap);
  610. u32 tmp;
  611. /* disable FIS reception */
  612. tmp = readl(port_mmio + PORT_CMD);
  613. tmp &= ~PORT_CMD_FIS_RX;
  614. writel(tmp, port_mmio + PORT_CMD);
  615. /* wait for completion, spec says 500ms, give it 1000 */
  616. tmp = ata_wait_register(ap, port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
  617. PORT_CMD_FIS_ON, 10, 1000);
  618. if (tmp & PORT_CMD_FIS_ON)
  619. return -EBUSY;
  620. return 0;
  621. }
  622. static void ahci_power_up(struct ata_port *ap)
  623. {
  624. struct ahci_host_priv *hpriv = ap->host->private_data;
  625. void __iomem *port_mmio = ahci_port_base(ap);
  626. u32 cmd;
  627. cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
  628. /* spin up device */
  629. if (hpriv->cap & HOST_CAP_SSS) {
  630. cmd |= PORT_CMD_SPIN_UP;
  631. writel(cmd, port_mmio + PORT_CMD);
  632. }
  633. /* wake up link */
  634. writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
  635. }
  636. static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  637. unsigned int hints)
  638. {
  639. struct ata_port *ap = link->ap;
  640. struct ahci_host_priv *hpriv = ap->host->private_data;
  641. struct ahci_port_priv *pp = ap->private_data;
  642. void __iomem *port_mmio = ahci_port_base(ap);
  643. if (policy != ATA_LPM_MAX_POWER) {
  644. /* wakeup flag only applies to the max power policy */
  645. hints &= ~ATA_LPM_WAKE_ONLY;
  646. /*
  647. * Disable interrupts on Phy Ready. This keeps us from
  648. * getting woken up due to spurious phy ready
  649. * interrupts.
  650. */
  651. pp->intr_mask &= ~PORT_IRQ_PHYRDY;
  652. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  653. sata_link_scr_lpm(link, policy, false);
  654. }
  655. if (hpriv->cap & HOST_CAP_ALPM) {
  656. u32 cmd = readl(port_mmio + PORT_CMD);
  657. if (policy == ATA_LPM_MAX_POWER || !(hints & ATA_LPM_HIPM)) {
  658. if (!(hints & ATA_LPM_WAKE_ONLY))
  659. cmd &= ~(PORT_CMD_ASP | PORT_CMD_ALPE);
  660. cmd |= PORT_CMD_ICC_ACTIVE;
  661. writel(cmd, port_mmio + PORT_CMD);
  662. readl(port_mmio + PORT_CMD);
  663. /* wait 10ms to be sure we've come out of LPM state */
  664. ata_msleep(ap, 10);
  665. if (hints & ATA_LPM_WAKE_ONLY)
  666. return 0;
  667. } else {
  668. cmd |= PORT_CMD_ALPE;
  669. if (policy == ATA_LPM_MIN_POWER)
  670. cmd |= PORT_CMD_ASP;
  671. else if (policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
  672. cmd &= ~PORT_CMD_ASP;
  673. /* write out new cmd value */
  674. writel(cmd, port_mmio + PORT_CMD);
  675. }
  676. }
  677. /* set aggressive device sleep */
  678. if ((hpriv->cap2 & HOST_CAP2_SDS) &&
  679. (hpriv->cap2 & HOST_CAP2_SADM) &&
  680. (link->device->flags & ATA_DFLAG_DEVSLP)) {
  681. if (policy == ATA_LPM_MIN_POWER ||
  682. policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
  683. ahci_set_aggressive_devslp(ap, true);
  684. else
  685. ahci_set_aggressive_devslp(ap, false);
  686. }
  687. if (policy == ATA_LPM_MAX_POWER) {
  688. sata_link_scr_lpm(link, policy, false);
  689. /* turn PHYRDY IRQ back on */
  690. pp->intr_mask |= PORT_IRQ_PHYRDY;
  691. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  692. }
  693. return 0;
  694. }
  695. #ifdef CONFIG_PM
  696. static void ahci_power_down(struct ata_port *ap)
  697. {
  698. struct ahci_host_priv *hpriv = ap->host->private_data;
  699. void __iomem *port_mmio = ahci_port_base(ap);
  700. u32 cmd, scontrol;
  701. if (!(hpriv->cap & HOST_CAP_SSS))
  702. return;
  703. /* put device into listen mode, first set PxSCTL.DET to 0 */
  704. scontrol = readl(port_mmio + PORT_SCR_CTL);
  705. scontrol &= ~0xf;
  706. writel(scontrol, port_mmio + PORT_SCR_CTL);
  707. /* then set PxCMD.SUD to 0 */
  708. cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
  709. cmd &= ~PORT_CMD_SPIN_UP;
  710. writel(cmd, port_mmio + PORT_CMD);
  711. }
  712. #endif
  713. static void ahci_start_port(struct ata_port *ap)
  714. {
  715. struct ahci_host_priv *hpriv = ap->host->private_data;
  716. struct ahci_port_priv *pp = ap->private_data;
  717. struct ata_link *link;
  718. struct ahci_em_priv *emp;
  719. ssize_t rc;
  720. int i;
  721. /* enable FIS reception */
  722. ahci_start_fis_rx(ap);
  723. /* enable DMA */
  724. if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))
  725. hpriv->start_engine(ap);
  726. /* turn on LEDs */
  727. if (ap->flags & ATA_FLAG_EM) {
  728. ata_for_each_link(link, ap, EDGE) {
  729. emp = &pp->em_priv[link->pmp];
  730. /* EM Transmit bit maybe busy during init */
  731. for (i = 0; i < EM_MAX_RETRY; i++) {
  732. rc = ap->ops->transmit_led_message(ap,
  733. emp->led_state,
  734. 4);
  735. /*
  736. * If busy, give a breather but do not
  737. * release EH ownership by using msleep()
  738. * instead of ata_msleep(). EM Transmit
  739. * bit is busy for the whole host and
  740. * releasing ownership will cause other
  741. * ports to fail the same way.
  742. */
  743. if (rc == -EBUSY)
  744. msleep(1);
  745. else
  746. break;
  747. }
  748. }
  749. }
  750. if (ap->flags & ATA_FLAG_SW_ACTIVITY)
  751. ata_for_each_link(link, ap, EDGE)
  752. ahci_init_sw_activity(link);
  753. }
  754. static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
  755. {
  756. int rc;
  757. struct ahci_host_priv *hpriv = ap->host->private_data;
  758. /* disable DMA */
  759. rc = hpriv->stop_engine(ap);
  760. if (rc) {
  761. *emsg = "failed to stop engine";
  762. return rc;
  763. }
  764. /* disable FIS reception */
  765. rc = ahci_stop_fis_rx(ap);
  766. if (rc) {
  767. *emsg = "failed stop FIS RX";
  768. return rc;
  769. }
  770. return 0;
  771. }
  772. int ahci_reset_controller(struct ata_host *host)
  773. {
  774. struct ahci_host_priv *hpriv = host->private_data;
  775. void __iomem *mmio = hpriv->mmio;
  776. u32 tmp;
  777. /* we must be in AHCI mode, before using anything
  778. * AHCI-specific, such as HOST_RESET.
  779. */
  780. ahci_enable_ahci(mmio);
  781. /* global controller reset */
  782. if (!ahci_skip_host_reset) {
  783. tmp = readl(mmio + HOST_CTL);
  784. if ((tmp & HOST_RESET) == 0) {
  785. writel(tmp | HOST_RESET, mmio + HOST_CTL);
  786. readl(mmio + HOST_CTL); /* flush */
  787. }
  788. /*
  789. * to perform host reset, OS should set HOST_RESET
  790. * and poll until this bit is read to be "0".
  791. * reset must complete within 1 second, or
  792. * the hardware should be considered fried.
  793. */
  794. tmp = ata_wait_register(NULL, mmio + HOST_CTL, HOST_RESET,
  795. HOST_RESET, 10, 1000);
  796. if (tmp & HOST_RESET) {
  797. dev_err(host->dev, "controller reset failed (0x%x)\n",
  798. tmp);
  799. return -EIO;
  800. }
  801. /* turn on AHCI mode */
  802. ahci_enable_ahci(mmio);
  803. /* Some registers might be cleared on reset. Restore
  804. * initial values.
  805. */
  806. if (!(hpriv->flags & AHCI_HFLAG_NO_WRITE_TO_RO))
  807. ahci_restore_initial_config(host);
  808. } else
  809. dev_info(host->dev, "skipping global host reset\n");
  810. return 0;
  811. }
  812. EXPORT_SYMBOL_GPL(ahci_reset_controller);
  813. static void ahci_sw_activity(struct ata_link *link)
  814. {
  815. struct ata_port *ap = link->ap;
  816. struct ahci_port_priv *pp = ap->private_data;
  817. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  818. if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
  819. return;
  820. emp->activity++;
  821. if (!timer_pending(&emp->timer))
  822. mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
  823. }
  824. static void ahci_sw_activity_blink(struct timer_list *t)
  825. {
  826. struct ahci_em_priv *emp = from_timer(emp, t, timer);
  827. struct ata_link *link = emp->link;
  828. struct ata_port *ap = link->ap;
  829. unsigned long led_message = emp->led_state;
  830. u32 activity_led_state;
  831. unsigned long flags;
  832. led_message &= EM_MSG_LED_VALUE;
  833. led_message |= ap->port_no | (link->pmp << 8);
  834. /* check to see if we've had activity. If so,
  835. * toggle state of LED and reset timer. If not,
  836. * turn LED to desired idle state.
  837. */
  838. spin_lock_irqsave(ap->lock, flags);
  839. if (emp->saved_activity != emp->activity) {
  840. emp->saved_activity = emp->activity;
  841. /* get the current LED state */
  842. activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
  843. if (activity_led_state)
  844. activity_led_state = 0;
  845. else
  846. activity_led_state = 1;
  847. /* clear old state */
  848. led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
  849. /* toggle state */
  850. led_message |= (activity_led_state << 16);
  851. mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
  852. } else {
  853. /* switch to idle */
  854. led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
  855. if (emp->blink_policy == BLINK_OFF)
  856. led_message |= (1 << 16);
  857. }
  858. spin_unlock_irqrestore(ap->lock, flags);
  859. ap->ops->transmit_led_message(ap, led_message, 4);
  860. }
  861. static void ahci_init_sw_activity(struct ata_link *link)
  862. {
  863. struct ata_port *ap = link->ap;
  864. struct ahci_port_priv *pp = ap->private_data;
  865. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  866. /* init activity stats, setup timer */
  867. emp->saved_activity = emp->activity = 0;
  868. emp->link = link;
  869. timer_setup(&emp->timer, ahci_sw_activity_blink, 0);
  870. /* check our blink policy and set flag for link if it's enabled */
  871. if (emp->blink_policy)
  872. link->flags |= ATA_LFLAG_SW_ACTIVITY;
  873. }
  874. int ahci_reset_em(struct ata_host *host)
  875. {
  876. struct ahci_host_priv *hpriv = host->private_data;
  877. void __iomem *mmio = hpriv->mmio;
  878. u32 em_ctl;
  879. em_ctl = readl(mmio + HOST_EM_CTL);
  880. if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
  881. return -EINVAL;
  882. writel(em_ctl | EM_CTL_RST, mmio + HOST_EM_CTL);
  883. return 0;
  884. }
  885. EXPORT_SYMBOL_GPL(ahci_reset_em);
  886. static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  887. ssize_t size)
  888. {
  889. struct ahci_host_priv *hpriv = ap->host->private_data;
  890. struct ahci_port_priv *pp = ap->private_data;
  891. void __iomem *mmio = hpriv->mmio;
  892. u32 em_ctl;
  893. u32 message[] = {0, 0};
  894. unsigned long flags;
  895. int pmp;
  896. struct ahci_em_priv *emp;
  897. /* get the slot number from the message */
  898. pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
  899. if (pmp < EM_MAX_SLOTS)
  900. emp = &pp->em_priv[pmp];
  901. else
  902. return -EINVAL;
  903. ahci_rpm_get_port(ap);
  904. spin_lock_irqsave(ap->lock, flags);
  905. /*
  906. * if we are still busy transmitting a previous message,
  907. * do not allow
  908. */
  909. em_ctl = readl(mmio + HOST_EM_CTL);
  910. if (em_ctl & EM_CTL_TM) {
  911. spin_unlock_irqrestore(ap->lock, flags);
  912. ahci_rpm_put_port(ap);
  913. return -EBUSY;
  914. }
  915. if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
  916. /*
  917. * create message header - this is all zero except for
  918. * the message size, which is 4 bytes.
  919. */
  920. message[0] |= (4 << 8);
  921. /* ignore 0:4 of byte zero, fill in port info yourself */
  922. message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
  923. /* write message to EM_LOC */
  924. writel(message[0], mmio + hpriv->em_loc);
  925. writel(message[1], mmio + hpriv->em_loc+4);
  926. /*
  927. * tell hardware to transmit the message
  928. */
  929. writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
  930. }
  931. /* save off new led state for port/slot */
  932. emp->led_state = state;
  933. spin_unlock_irqrestore(ap->lock, flags);
  934. ahci_rpm_put_port(ap);
  935. return size;
  936. }
  937. static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
  938. {
  939. struct ahci_port_priv *pp = ap->private_data;
  940. struct ata_link *link;
  941. struct ahci_em_priv *emp;
  942. int rc = 0;
  943. ata_for_each_link(link, ap, EDGE) {
  944. emp = &pp->em_priv[link->pmp];
  945. rc += sprintf(buf, "%lx\n", emp->led_state);
  946. }
  947. return rc;
  948. }
  949. static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  950. size_t size)
  951. {
  952. unsigned int state;
  953. int pmp;
  954. struct ahci_port_priv *pp = ap->private_data;
  955. struct ahci_em_priv *emp;
  956. if (kstrtouint(buf, 0, &state) < 0)
  957. return -EINVAL;
  958. /* get the slot number from the message */
  959. pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
  960. if (pmp < EM_MAX_SLOTS) {
  961. pmp = array_index_nospec(pmp, EM_MAX_SLOTS);
  962. emp = &pp->em_priv[pmp];
  963. } else {
  964. return -EINVAL;
  965. }
  966. /* mask off the activity bits if we are in sw_activity
  967. * mode, user should turn off sw_activity before setting
  968. * activity led through em_message
  969. */
  970. if (emp->blink_policy)
  971. state &= ~EM_MSG_LED_VALUE_ACTIVITY;
  972. return ap->ops->transmit_led_message(ap, state, size);
  973. }
  974. static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
  975. {
  976. struct ata_link *link = dev->link;
  977. struct ata_port *ap = link->ap;
  978. struct ahci_port_priv *pp = ap->private_data;
  979. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  980. u32 port_led_state = emp->led_state;
  981. /* save the desired Activity LED behavior */
  982. if (val == OFF) {
  983. /* clear LFLAG */
  984. link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
  985. /* set the LED to OFF */
  986. port_led_state &= EM_MSG_LED_VALUE_OFF;
  987. port_led_state |= (ap->port_no | (link->pmp << 8));
  988. ap->ops->transmit_led_message(ap, port_led_state, 4);
  989. } else {
  990. link->flags |= ATA_LFLAG_SW_ACTIVITY;
  991. if (val == BLINK_OFF) {
  992. /* set LED to ON for idle */
  993. port_led_state &= EM_MSG_LED_VALUE_OFF;
  994. port_led_state |= (ap->port_no | (link->pmp << 8));
  995. port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
  996. ap->ops->transmit_led_message(ap, port_led_state, 4);
  997. }
  998. }
  999. emp->blink_policy = val;
  1000. return 0;
  1001. }
  1002. static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
  1003. {
  1004. struct ata_link *link = dev->link;
  1005. struct ata_port *ap = link->ap;
  1006. struct ahci_port_priv *pp = ap->private_data;
  1007. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  1008. /* display the saved value of activity behavior for this
  1009. * disk.
  1010. */
  1011. return sprintf(buf, "%d\n", emp->blink_policy);
  1012. }
  1013. static void ahci_port_init(struct device *dev, struct ata_port *ap,
  1014. int port_no, void __iomem *mmio,
  1015. void __iomem *port_mmio)
  1016. {
  1017. struct ahci_host_priv *hpriv = ap->host->private_data;
  1018. const char *emsg = NULL;
  1019. int rc;
  1020. u32 tmp;
  1021. /* make sure port is not active */
  1022. rc = ahci_deinit_port(ap, &emsg);
  1023. if (rc)
  1024. dev_warn(dev, "%s (%d)\n", emsg, rc);
  1025. /* clear SError */
  1026. tmp = readl(port_mmio + PORT_SCR_ERR);
  1027. VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
  1028. writel(tmp, port_mmio + PORT_SCR_ERR);
  1029. /* clear port IRQ */
  1030. tmp = readl(port_mmio + PORT_IRQ_STAT);
  1031. VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
  1032. if (tmp)
  1033. writel(tmp, port_mmio + PORT_IRQ_STAT);
  1034. writel(1 << port_no, mmio + HOST_IRQ_STAT);
  1035. /* mark esata ports */
  1036. tmp = readl(port_mmio + PORT_CMD);
  1037. if ((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS))
  1038. ap->pflags |= ATA_PFLAG_EXTERNAL;
  1039. }
  1040. void ahci_init_controller(struct ata_host *host)
  1041. {
  1042. struct ahci_host_priv *hpriv = host->private_data;
  1043. void __iomem *mmio = hpriv->mmio;
  1044. int i;
  1045. void __iomem *port_mmio;
  1046. u32 tmp;
  1047. for (i = 0; i < host->n_ports; i++) {
  1048. struct ata_port *ap = host->ports[i];
  1049. port_mmio = ahci_port_base(ap);
  1050. if (ata_port_is_dummy(ap))
  1051. continue;
  1052. ahci_port_init(host->dev, ap, i, mmio, port_mmio);
  1053. }
  1054. tmp = readl(mmio + HOST_CTL);
  1055. VPRINTK("HOST_CTL 0x%x\n", tmp);
  1056. writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
  1057. tmp = readl(mmio + HOST_CTL);
  1058. VPRINTK("HOST_CTL 0x%x\n", tmp);
  1059. }
  1060. EXPORT_SYMBOL_GPL(ahci_init_controller);
  1061. static void ahci_dev_config(struct ata_device *dev)
  1062. {
  1063. struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
  1064. if (hpriv->flags & AHCI_HFLAG_SECT255) {
  1065. dev->max_sectors = 255;
  1066. ata_dev_info(dev,
  1067. "SB600 AHCI: limiting to 255 sectors per cmd\n");
  1068. }
  1069. }
  1070. unsigned int ahci_dev_classify(struct ata_port *ap)
  1071. {
  1072. void __iomem *port_mmio = ahci_port_base(ap);
  1073. struct ata_taskfile tf;
  1074. u32 tmp;
  1075. tmp = readl(port_mmio + PORT_SIG);
  1076. tf.lbah = (tmp >> 24) & 0xff;
  1077. tf.lbam = (tmp >> 16) & 0xff;
  1078. tf.lbal = (tmp >> 8) & 0xff;
  1079. tf.nsect = (tmp) & 0xff;
  1080. return ata_dev_classify(&tf);
  1081. }
  1082. EXPORT_SYMBOL_GPL(ahci_dev_classify);
  1083. void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
  1084. u32 opts)
  1085. {
  1086. dma_addr_t cmd_tbl_dma;
  1087. cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
  1088. pp->cmd_slot[tag].opts = cpu_to_le32(opts);
  1089. pp->cmd_slot[tag].status = 0;
  1090. pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
  1091. pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
  1092. }
  1093. EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
  1094. int ahci_kick_engine(struct ata_port *ap)
  1095. {
  1096. void __iomem *port_mmio = ahci_port_base(ap);
  1097. struct ahci_host_priv *hpriv = ap->host->private_data;
  1098. u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
  1099. u32 tmp;
  1100. int busy, rc;
  1101. /* stop engine */
  1102. rc = hpriv->stop_engine(ap);
  1103. if (rc)
  1104. goto out_restart;
  1105. /* need to do CLO?
  1106. * always do CLO if PMP is attached (AHCI-1.3 9.2)
  1107. */
  1108. busy = status & (ATA_BUSY | ATA_DRQ);
  1109. if (!busy && !sata_pmp_attached(ap)) {
  1110. rc = 0;
  1111. goto out_restart;
  1112. }
  1113. if (!(hpriv->cap & HOST_CAP_CLO)) {
  1114. rc = -EOPNOTSUPP;
  1115. goto out_restart;
  1116. }
  1117. /* perform CLO */
  1118. tmp = readl(port_mmio + PORT_CMD);
  1119. tmp |= PORT_CMD_CLO;
  1120. writel(tmp, port_mmio + PORT_CMD);
  1121. rc = 0;
  1122. tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
  1123. PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
  1124. if (tmp & PORT_CMD_CLO)
  1125. rc = -EIO;
  1126. /* restart engine */
  1127. out_restart:
  1128. hpriv->start_engine(ap);
  1129. return rc;
  1130. }
  1131. EXPORT_SYMBOL_GPL(ahci_kick_engine);
  1132. static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
  1133. struct ata_taskfile *tf, int is_cmd, u16 flags,
  1134. unsigned long timeout_msec)
  1135. {
  1136. const u32 cmd_fis_len = 5; /* five dwords */
  1137. struct ahci_port_priv *pp = ap->private_data;
  1138. void __iomem *port_mmio = ahci_port_base(ap);
  1139. u8 *fis = pp->cmd_tbl;
  1140. u32 tmp;
  1141. /* prep the command */
  1142. ata_tf_to_fis(tf, pmp, is_cmd, fis);
  1143. ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
  1144. /* set port value for softreset of Port Multiplier */
  1145. if (pp->fbs_enabled && pp->fbs_last_dev != pmp) {
  1146. tmp = readl(port_mmio + PORT_FBS);
  1147. tmp &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
  1148. tmp |= pmp << PORT_FBS_DEV_OFFSET;
  1149. writel(tmp, port_mmio + PORT_FBS);
  1150. pp->fbs_last_dev = pmp;
  1151. }
  1152. /* issue & wait */
  1153. writel(1, port_mmio + PORT_CMD_ISSUE);
  1154. if (timeout_msec) {
  1155. tmp = ata_wait_register(ap, port_mmio + PORT_CMD_ISSUE,
  1156. 0x1, 0x1, 1, timeout_msec);
  1157. if (tmp & 0x1) {
  1158. ahci_kick_engine(ap);
  1159. return -EBUSY;
  1160. }
  1161. } else
  1162. readl(port_mmio + PORT_CMD_ISSUE); /* flush */
  1163. return 0;
  1164. }
  1165. int ahci_do_softreset(struct ata_link *link, unsigned int *class,
  1166. int pmp, unsigned long deadline,
  1167. int (*check_ready)(struct ata_link *link))
  1168. {
  1169. struct ata_port *ap = link->ap;
  1170. struct ahci_host_priv *hpriv = ap->host->private_data;
  1171. struct ahci_port_priv *pp = ap->private_data;
  1172. const char *reason = NULL;
  1173. unsigned long now, msecs;
  1174. struct ata_taskfile tf;
  1175. bool fbs_disabled = false;
  1176. int rc;
  1177. DPRINTK("ENTER\n");
  1178. /* prepare for SRST (AHCI-1.1 10.4.1) */
  1179. rc = ahci_kick_engine(ap);
  1180. if (rc && rc != -EOPNOTSUPP)
  1181. ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
  1182. /*
  1183. * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
  1184. * clear PxFBS.EN to '0' prior to issuing software reset to devices
  1185. * that is attached to port multiplier.
  1186. */
  1187. if (!ata_is_host_link(link) && pp->fbs_enabled) {
  1188. ahci_disable_fbs(ap);
  1189. fbs_disabled = true;
  1190. }
  1191. ata_tf_init(link->device, &tf);
  1192. /* issue the first H2D Register FIS */
  1193. msecs = 0;
  1194. now = jiffies;
  1195. if (time_after(deadline, now))
  1196. msecs = jiffies_to_msecs(deadline - now);
  1197. tf.ctl |= ATA_SRST;
  1198. if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
  1199. AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
  1200. rc = -EIO;
  1201. reason = "1st FIS failed";
  1202. goto fail;
  1203. }
  1204. /* spec says at least 5us, but be generous and sleep for 1ms */
  1205. ata_msleep(ap, 1);
  1206. /* issue the second H2D Register FIS */
  1207. tf.ctl &= ~ATA_SRST;
  1208. ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
  1209. /* wait for link to become ready */
  1210. rc = ata_wait_after_reset(link, deadline, check_ready);
  1211. if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
  1212. /*
  1213. * Workaround for cases where link online status can't
  1214. * be trusted. Treat device readiness timeout as link
  1215. * offline.
  1216. */
  1217. ata_link_info(link, "device not ready, treating as offline\n");
  1218. *class = ATA_DEV_NONE;
  1219. } else if (rc) {
  1220. /* link occupied, -ENODEV too is an error */
  1221. reason = "device not ready";
  1222. goto fail;
  1223. } else
  1224. *class = ahci_dev_classify(ap);
  1225. /* re-enable FBS if disabled before */
  1226. if (fbs_disabled)
  1227. ahci_enable_fbs(ap);
  1228. DPRINTK("EXIT, class=%u\n", *class);
  1229. return 0;
  1230. fail:
  1231. ata_link_err(link, "softreset failed (%s)\n", reason);
  1232. return rc;
  1233. }
  1234. int ahci_check_ready(struct ata_link *link)
  1235. {
  1236. void __iomem *port_mmio = ahci_port_base(link->ap);
  1237. u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
  1238. return ata_check_ready(status);
  1239. }
  1240. EXPORT_SYMBOL_GPL(ahci_check_ready);
  1241. static int ahci_softreset(struct ata_link *link, unsigned int *class,
  1242. unsigned long deadline)
  1243. {
  1244. int pmp = sata_srst_pmp(link);
  1245. DPRINTK("ENTER\n");
  1246. return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
  1247. }
  1248. EXPORT_SYMBOL_GPL(ahci_do_softreset);
  1249. static int ahci_bad_pmp_check_ready(struct ata_link *link)
  1250. {
  1251. void __iomem *port_mmio = ahci_port_base(link->ap);
  1252. u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
  1253. u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
  1254. /*
  1255. * There is no need to check TFDATA if BAD PMP is found due to HW bug,
  1256. * which can save timeout delay.
  1257. */
  1258. if (irq_status & PORT_IRQ_BAD_PMP)
  1259. return -EIO;
  1260. return ata_check_ready(status);
  1261. }
  1262. static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  1263. unsigned long deadline)
  1264. {
  1265. struct ata_port *ap = link->ap;
  1266. void __iomem *port_mmio = ahci_port_base(ap);
  1267. int pmp = sata_srst_pmp(link);
  1268. int rc;
  1269. u32 irq_sts;
  1270. DPRINTK("ENTER\n");
  1271. rc = ahci_do_softreset(link, class, pmp, deadline,
  1272. ahci_bad_pmp_check_ready);
  1273. /*
  1274. * Soft reset fails with IPMS set when PMP is enabled but
  1275. * SATA HDD/ODD is connected to SATA port, do soft reset
  1276. * again to port 0.
  1277. */
  1278. if (rc == -EIO) {
  1279. irq_sts = readl(port_mmio + PORT_IRQ_STAT);
  1280. if (irq_sts & PORT_IRQ_BAD_PMP) {
  1281. ata_link_warn(link,
  1282. "applying PMP SRST workaround "
  1283. "and retrying\n");
  1284. rc = ahci_do_softreset(link, class, 0, deadline,
  1285. ahci_check_ready);
  1286. }
  1287. }
  1288. return rc;
  1289. }
  1290. int ahci_do_hardreset(struct ata_link *link, unsigned int *class,
  1291. unsigned long deadline, bool *online)
  1292. {
  1293. const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
  1294. struct ata_port *ap = link->ap;
  1295. struct ahci_port_priv *pp = ap->private_data;
  1296. struct ahci_host_priv *hpriv = ap->host->private_data;
  1297. u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
  1298. struct ata_taskfile tf;
  1299. int rc;
  1300. DPRINTK("ENTER\n");
  1301. hpriv->stop_engine(ap);
  1302. /* clear D2H reception area to properly wait for D2H FIS */
  1303. ata_tf_init(link->device, &tf);
  1304. tf.command = ATA_BUSY;
  1305. ata_tf_to_fis(&tf, 0, 0, d2h_fis);
  1306. rc = sata_link_hardreset(link, timing, deadline, online,
  1307. ahci_check_ready);
  1308. hpriv->start_engine(ap);
  1309. if (*online)
  1310. *class = ahci_dev_classify(ap);
  1311. DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
  1312. return rc;
  1313. }
  1314. EXPORT_SYMBOL_GPL(ahci_do_hardreset);
  1315. static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  1316. unsigned long deadline)
  1317. {
  1318. bool online;
  1319. return ahci_do_hardreset(link, class, deadline, &online);
  1320. }
  1321. static void ahci_postreset(struct ata_link *link, unsigned int *class)
  1322. {
  1323. struct ata_port *ap = link->ap;
  1324. void __iomem *port_mmio = ahci_port_base(ap);
  1325. u32 new_tmp, tmp;
  1326. ata_std_postreset(link, class);
  1327. /* Make sure port's ATAPI bit is set appropriately */
  1328. new_tmp = tmp = readl(port_mmio + PORT_CMD);
  1329. if (*class == ATA_DEV_ATAPI)
  1330. new_tmp |= PORT_CMD_ATAPI;
  1331. else
  1332. new_tmp &= ~PORT_CMD_ATAPI;
  1333. if (new_tmp != tmp) {
  1334. writel(new_tmp, port_mmio + PORT_CMD);
  1335. readl(port_mmio + PORT_CMD); /* flush */
  1336. }
  1337. }
  1338. static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
  1339. {
  1340. struct scatterlist *sg;
  1341. struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
  1342. unsigned int si;
  1343. VPRINTK("ENTER\n");
  1344. /*
  1345. * Next, the S/G list.
  1346. */
  1347. for_each_sg(qc->sg, sg, qc->n_elem, si) {
  1348. dma_addr_t addr = sg_dma_address(sg);
  1349. u32 sg_len = sg_dma_len(sg);
  1350. ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
  1351. ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
  1352. ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
  1353. }
  1354. return si;
  1355. }
  1356. static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
  1357. {
  1358. struct ata_port *ap = qc->ap;
  1359. struct ahci_port_priv *pp = ap->private_data;
  1360. if (!sata_pmp_attached(ap) || pp->fbs_enabled)
  1361. return ata_std_qc_defer(qc);
  1362. else
  1363. return sata_pmp_qc_defer_cmd_switch(qc);
  1364. }
  1365. static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc)
  1366. {
  1367. struct ata_port *ap = qc->ap;
  1368. struct ahci_port_priv *pp = ap->private_data;
  1369. int is_atapi = ata_is_atapi(qc->tf.protocol);
  1370. void *cmd_tbl;
  1371. u32 opts;
  1372. const u32 cmd_fis_len = 5; /* five dwords */
  1373. unsigned int n_elem;
  1374. /*
  1375. * Fill in command table information. First, the header,
  1376. * a SATA Register - Host to Device command FIS.
  1377. */
  1378. cmd_tbl = pp->cmd_tbl + qc->hw_tag * AHCI_CMD_TBL_SZ;
  1379. ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
  1380. if (is_atapi) {
  1381. memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
  1382. memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
  1383. }
  1384. n_elem = 0;
  1385. if (qc->flags & ATA_QCFLAG_DMAMAP)
  1386. n_elem = ahci_fill_sg(qc, cmd_tbl);
  1387. /*
  1388. * Fill in command slot information.
  1389. */
  1390. opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
  1391. if (qc->tf.flags & ATA_TFLAG_WRITE)
  1392. opts |= AHCI_CMD_WRITE;
  1393. if (is_atapi)
  1394. opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
  1395. ahci_fill_cmd_slot(pp, qc->hw_tag, opts);
  1396. return AC_ERR_OK;
  1397. }
  1398. static void ahci_fbs_dec_intr(struct ata_port *ap)
  1399. {
  1400. struct ahci_port_priv *pp = ap->private_data;
  1401. void __iomem *port_mmio = ahci_port_base(ap);
  1402. u32 fbs = readl(port_mmio + PORT_FBS);
  1403. int retries = 3;
  1404. DPRINTK("ENTER\n");
  1405. BUG_ON(!pp->fbs_enabled);
  1406. /* time to wait for DEC is not specified by AHCI spec,
  1407. * add a retry loop for safety.
  1408. */
  1409. writel(fbs | PORT_FBS_DEC, port_mmio + PORT_FBS);
  1410. fbs = readl(port_mmio + PORT_FBS);
  1411. while ((fbs & PORT_FBS_DEC) && retries--) {
  1412. udelay(1);
  1413. fbs = readl(port_mmio + PORT_FBS);
  1414. }
  1415. if (fbs & PORT_FBS_DEC)
  1416. dev_err(ap->host->dev, "failed to clear device error\n");
  1417. }
  1418. static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
  1419. {
  1420. struct ahci_host_priv *hpriv = ap->host->private_data;
  1421. struct ahci_port_priv *pp = ap->private_data;
  1422. struct ata_eh_info *host_ehi = &ap->link.eh_info;
  1423. struct ata_link *link = NULL;
  1424. struct ata_queued_cmd *active_qc;
  1425. struct ata_eh_info *active_ehi;
  1426. bool fbs_need_dec = false;
  1427. u32 serror;
  1428. /* determine active link with error */
  1429. if (pp->fbs_enabled) {
  1430. void __iomem *port_mmio = ahci_port_base(ap);
  1431. u32 fbs = readl(port_mmio + PORT_FBS);
  1432. int pmp = fbs >> PORT_FBS_DWE_OFFSET;
  1433. if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links)) {
  1434. link = &ap->pmp_link[pmp];
  1435. fbs_need_dec = true;
  1436. }
  1437. } else
  1438. ata_for_each_link(link, ap, EDGE)
  1439. if (ata_link_active(link))
  1440. break;
  1441. if (!link)
  1442. link = &ap->link;
  1443. active_qc = ata_qc_from_tag(ap, link->active_tag);
  1444. active_ehi = &link->eh_info;
  1445. /* record irq stat */
  1446. ata_ehi_clear_desc(host_ehi);
  1447. ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
  1448. /* AHCI needs SError cleared; otherwise, it might lock up */
  1449. ahci_scr_read(&ap->link, SCR_ERROR, &serror);
  1450. ahci_scr_write(&ap->link, SCR_ERROR, serror);
  1451. host_ehi->serror |= serror;
  1452. /* some controllers set IRQ_IF_ERR on device errors, ignore it */
  1453. if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
  1454. irq_stat &= ~PORT_IRQ_IF_ERR;
  1455. if (irq_stat & PORT_IRQ_TF_ERR) {
  1456. /* If qc is active, charge it; otherwise, the active
  1457. * link. There's no active qc on NCQ errors. It will
  1458. * be determined by EH by reading log page 10h.
  1459. */
  1460. if (active_qc)
  1461. active_qc->err_mask |= AC_ERR_DEV;
  1462. else
  1463. active_ehi->err_mask |= AC_ERR_DEV;
  1464. if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
  1465. host_ehi->serror &= ~SERR_INTERNAL;
  1466. }
  1467. if (irq_stat & PORT_IRQ_UNK_FIS) {
  1468. u32 *unk = pp->rx_fis + RX_FIS_UNK;
  1469. active_ehi->err_mask |= AC_ERR_HSM;
  1470. active_ehi->action |= ATA_EH_RESET;
  1471. ata_ehi_push_desc(active_ehi,
  1472. "unknown FIS %08x %08x %08x %08x" ,
  1473. unk[0], unk[1], unk[2], unk[3]);
  1474. }
  1475. if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
  1476. active_ehi->err_mask |= AC_ERR_HSM;
  1477. active_ehi->action |= ATA_EH_RESET;
  1478. ata_ehi_push_desc(active_ehi, "incorrect PMP");
  1479. }
  1480. if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
  1481. host_ehi->err_mask |= AC_ERR_HOST_BUS;
  1482. host_ehi->action |= ATA_EH_RESET;
  1483. ata_ehi_push_desc(host_ehi, "host bus error");
  1484. }
  1485. if (irq_stat & PORT_IRQ_IF_ERR) {
  1486. if (fbs_need_dec)
  1487. active_ehi->err_mask |= AC_ERR_DEV;
  1488. else {
  1489. host_ehi->err_mask |= AC_ERR_ATA_BUS;
  1490. host_ehi->action |= ATA_EH_RESET;
  1491. }
  1492. ata_ehi_push_desc(host_ehi, "interface fatal error");
  1493. }
  1494. if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
  1495. ata_ehi_hotplugged(host_ehi);
  1496. ata_ehi_push_desc(host_ehi, "%s",
  1497. irq_stat & PORT_IRQ_CONNECT ?
  1498. "connection status changed" : "PHY RDY changed");
  1499. }
  1500. /* okay, let's hand over to EH */
  1501. if (irq_stat & PORT_IRQ_FREEZE)
  1502. ata_port_freeze(ap);
  1503. else if (fbs_need_dec) {
  1504. ata_link_abort(link);
  1505. ahci_fbs_dec_intr(ap);
  1506. } else
  1507. ata_port_abort(ap);
  1508. }
  1509. static void ahci_handle_port_interrupt(struct ata_port *ap,
  1510. void __iomem *port_mmio, u32 status)
  1511. {
  1512. struct ata_eh_info *ehi = &ap->link.eh_info;
  1513. struct ahci_port_priv *pp = ap->private_data;
  1514. struct ahci_host_priv *hpriv = ap->host->private_data;
  1515. int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
  1516. u32 qc_active = 0;
  1517. int rc;
  1518. /* ignore BAD_PMP while resetting */
  1519. if (unlikely(resetting))
  1520. status &= ~PORT_IRQ_BAD_PMP;
  1521. if (sata_lpm_ignore_phy_events(&ap->link)) {
  1522. status &= ~PORT_IRQ_PHYRDY;
  1523. ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);
  1524. }
  1525. if (unlikely(status & PORT_IRQ_ERROR)) {
  1526. ahci_error_intr(ap, status);
  1527. return;
  1528. }
  1529. if (status & PORT_IRQ_SDB_FIS) {
  1530. /* If SNotification is available, leave notification
  1531. * handling to sata_async_notification(). If not,
  1532. * emulate it by snooping SDB FIS RX area.
  1533. *
  1534. * Snooping FIS RX area is probably cheaper than
  1535. * poking SNotification but some constrollers which
  1536. * implement SNotification, ICH9 for example, don't
  1537. * store AN SDB FIS into receive area.
  1538. */
  1539. if (hpriv->cap & HOST_CAP_SNTF)
  1540. sata_async_notification(ap);
  1541. else {
  1542. /* If the 'N' bit in word 0 of the FIS is set,
  1543. * we just received asynchronous notification.
  1544. * Tell libata about it.
  1545. *
  1546. * Lack of SNotification should not appear in
  1547. * ahci 1.2, so the workaround is unnecessary
  1548. * when FBS is enabled.
  1549. */
  1550. if (pp->fbs_enabled)
  1551. WARN_ON_ONCE(1);
  1552. else {
  1553. const __le32 *f = pp->rx_fis + RX_FIS_SDB;
  1554. u32 f0 = le32_to_cpu(f[0]);
  1555. if (f0 & (1 << 15))
  1556. sata_async_notification(ap);
  1557. }
  1558. }
  1559. }
  1560. /* pp->active_link is not reliable once FBS is enabled, both
  1561. * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
  1562. * NCQ and non-NCQ commands may be in flight at the same time.
  1563. */
  1564. if (pp->fbs_enabled) {
  1565. if (ap->qc_active) {
  1566. qc_active = readl(port_mmio + PORT_SCR_ACT);
  1567. qc_active |= readl(port_mmio + PORT_CMD_ISSUE);
  1568. }
  1569. } else {
  1570. /* pp->active_link is valid iff any command is in flight */
  1571. if (ap->qc_active && pp->active_link->sactive)
  1572. qc_active = readl(port_mmio + PORT_SCR_ACT);
  1573. else
  1574. qc_active = readl(port_mmio + PORT_CMD_ISSUE);
  1575. }
  1576. rc = ata_qc_complete_multiple(ap, qc_active);
  1577. /* while resetting, invalid completions are expected */
  1578. if (unlikely(rc < 0 && !resetting)) {
  1579. ehi->err_mask |= AC_ERR_HSM;
  1580. ehi->action |= ATA_EH_RESET;
  1581. ata_port_freeze(ap);
  1582. }
  1583. }
  1584. static void ahci_port_intr(struct ata_port *ap)
  1585. {
  1586. void __iomem *port_mmio = ahci_port_base(ap);
  1587. u32 status;
  1588. status = readl(port_mmio + PORT_IRQ_STAT);
  1589. writel(status, port_mmio + PORT_IRQ_STAT);
  1590. ahci_handle_port_interrupt(ap, port_mmio, status);
  1591. }
  1592. static irqreturn_t ahci_multi_irqs_intr_hard(int irq, void *dev_instance)
  1593. {
  1594. struct ata_port *ap = dev_instance;
  1595. void __iomem *port_mmio = ahci_port_base(ap);
  1596. u32 status;
  1597. VPRINTK("ENTER\n");
  1598. status = readl(port_mmio + PORT_IRQ_STAT);
  1599. writel(status, port_mmio + PORT_IRQ_STAT);
  1600. spin_lock(ap->lock);
  1601. ahci_handle_port_interrupt(ap, port_mmio, status);
  1602. spin_unlock(ap->lock);
  1603. VPRINTK("EXIT\n");
  1604. return IRQ_HANDLED;
  1605. }
  1606. u32 ahci_handle_port_intr(struct ata_host *host, u32 irq_masked)
  1607. {
  1608. unsigned int i, handled = 0;
  1609. for (i = 0; i < host->n_ports; i++) {
  1610. struct ata_port *ap;
  1611. if (!(irq_masked & (1 << i)))
  1612. continue;
  1613. ap = host->ports[i];
  1614. if (ap) {
  1615. ahci_port_intr(ap);
  1616. VPRINTK("port %u\n", i);
  1617. } else {
  1618. VPRINTK("port %u (no irq)\n", i);
  1619. if (ata_ratelimit())
  1620. dev_warn(host->dev,
  1621. "interrupt on disabled port %u\n", i);
  1622. }
  1623. handled = 1;
  1624. }
  1625. return handled;
  1626. }
  1627. EXPORT_SYMBOL_GPL(ahci_handle_port_intr);
  1628. static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance)
  1629. {
  1630. struct ata_host *host = dev_instance;
  1631. struct ahci_host_priv *hpriv;
  1632. unsigned int rc = 0;
  1633. void __iomem *mmio;
  1634. u32 irq_stat, irq_masked;
  1635. VPRINTK("ENTER\n");
  1636. hpriv = host->private_data;
  1637. mmio = hpriv->mmio;
  1638. /* sigh. 0xffffffff is a valid return from h/w */
  1639. irq_stat = readl(mmio + HOST_IRQ_STAT);
  1640. if (!irq_stat)
  1641. return IRQ_NONE;
  1642. irq_masked = irq_stat & hpriv->port_map;
  1643. spin_lock(&host->lock);
  1644. rc = ahci_handle_port_intr(host, irq_masked);
  1645. /* HOST_IRQ_STAT behaves as level triggered latch meaning that
  1646. * it should be cleared after all the port events are cleared;
  1647. * otherwise, it will raise a spurious interrupt after each
  1648. * valid one. Please read section 10.6.2 of ahci 1.1 for more
  1649. * information.
  1650. *
  1651. * Also, use the unmasked value to clear interrupt as spurious
  1652. * pending event on a dummy port might cause screaming IRQ.
  1653. */
  1654. writel(irq_stat, mmio + HOST_IRQ_STAT);
  1655. spin_unlock(&host->lock);
  1656. VPRINTK("EXIT\n");
  1657. return IRQ_RETVAL(rc);
  1658. }
  1659. unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
  1660. {
  1661. struct ata_port *ap = qc->ap;
  1662. void __iomem *port_mmio = ahci_port_base(ap);
  1663. struct ahci_port_priv *pp = ap->private_data;
  1664. /* Keep track of the currently active link. It will be used
  1665. * in completion path to determine whether NCQ phase is in
  1666. * progress.
  1667. */
  1668. pp->active_link = qc->dev->link;
  1669. if (ata_is_ncq(qc->tf.protocol))
  1670. writel(1 << qc->hw_tag, port_mmio + PORT_SCR_ACT);
  1671. if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
  1672. u32 fbs = readl(port_mmio + PORT_FBS);
  1673. fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
  1674. fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
  1675. writel(fbs, port_mmio + PORT_FBS);
  1676. pp->fbs_last_dev = qc->dev->link->pmp;
  1677. }
  1678. writel(1 << qc->hw_tag, port_mmio + PORT_CMD_ISSUE);
  1679. ahci_sw_activity(qc->dev->link);
  1680. return 0;
  1681. }
  1682. EXPORT_SYMBOL_GPL(ahci_qc_issue);
  1683. static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
  1684. {
  1685. struct ahci_port_priv *pp = qc->ap->private_data;
  1686. u8 *rx_fis = pp->rx_fis;
  1687. if (pp->fbs_enabled)
  1688. rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
  1689. /*
  1690. * After a successful execution of an ATA PIO data-in command,
  1691. * the device doesn't send D2H Reg FIS to update the TF and
  1692. * the host should take TF and E_Status from the preceding PIO
  1693. * Setup FIS.
  1694. */
  1695. if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
  1696. !(qc->flags & ATA_QCFLAG_FAILED)) {
  1697. ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);
  1698. qc->result_tf.command = (rx_fis + RX_FIS_PIO_SETUP)[15];
  1699. } else
  1700. ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);
  1701. return true;
  1702. }
  1703. static void ahci_freeze(struct ata_port *ap)
  1704. {
  1705. void __iomem *port_mmio = ahci_port_base(ap);
  1706. /* turn IRQ off */
  1707. writel(0, port_mmio + PORT_IRQ_MASK);
  1708. }
  1709. static void ahci_thaw(struct ata_port *ap)
  1710. {
  1711. struct ahci_host_priv *hpriv = ap->host->private_data;
  1712. void __iomem *mmio = hpriv->mmio;
  1713. void __iomem *port_mmio = ahci_port_base(ap);
  1714. u32 tmp;
  1715. struct ahci_port_priv *pp = ap->private_data;
  1716. /* clear IRQ */
  1717. tmp = readl(port_mmio + PORT_IRQ_STAT);
  1718. writel(tmp, port_mmio + PORT_IRQ_STAT);
  1719. writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
  1720. /* turn IRQ back on */
  1721. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  1722. }
  1723. void ahci_error_handler(struct ata_port *ap)
  1724. {
  1725. struct ahci_host_priv *hpriv = ap->host->private_data;
  1726. if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
  1727. /* restart engine */
  1728. hpriv->stop_engine(ap);
  1729. hpriv->start_engine(ap);
  1730. }
  1731. sata_pmp_error_handler(ap);
  1732. if (!ata_dev_enabled(ap->link.device))
  1733. hpriv->stop_engine(ap);
  1734. }
  1735. EXPORT_SYMBOL_GPL(ahci_error_handler);
  1736. static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
  1737. {
  1738. struct ata_port *ap = qc->ap;
  1739. /* make DMA engine forget about the failed command */
  1740. if (qc->flags & ATA_QCFLAG_FAILED)
  1741. ahci_kick_engine(ap);
  1742. }
  1743. static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep)
  1744. {
  1745. struct ahci_host_priv *hpriv = ap->host->private_data;
  1746. void __iomem *port_mmio = ahci_port_base(ap);
  1747. struct ata_device *dev = ap->link.device;
  1748. u32 devslp, dm, dito, mdat, deto, dito_conf;
  1749. int rc;
  1750. unsigned int err_mask;
  1751. devslp = readl(port_mmio + PORT_DEVSLP);
  1752. if (!(devslp & PORT_DEVSLP_DSP)) {
  1753. dev_info(ap->host->dev, "port does not support device sleep\n");
  1754. return;
  1755. }
  1756. /* disable device sleep */
  1757. if (!sleep) {
  1758. if (devslp & PORT_DEVSLP_ADSE) {
  1759. writel(devslp & ~PORT_DEVSLP_ADSE,
  1760. port_mmio + PORT_DEVSLP);
  1761. err_mask = ata_dev_set_feature(dev,
  1762. SETFEATURES_SATA_DISABLE,
  1763. SATA_DEVSLP);
  1764. if (err_mask && err_mask != AC_ERR_DEV)
  1765. ata_dev_warn(dev, "failed to disable DEVSLP\n");
  1766. }
  1767. return;
  1768. }
  1769. dm = (devslp & PORT_DEVSLP_DM_MASK) >> PORT_DEVSLP_DM_OFFSET;
  1770. dito = devslp_idle_timeout / (dm + 1);
  1771. if (dito > 0x3ff)
  1772. dito = 0x3ff;
  1773. dito_conf = (devslp >> PORT_DEVSLP_DITO_OFFSET) & 0x3FF;
  1774. /* device sleep was already enabled and same dito */
  1775. if ((devslp & PORT_DEVSLP_ADSE) && (dito_conf == dito))
  1776. return;
  1777. /* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
  1778. rc = hpriv->stop_engine(ap);
  1779. if (rc)
  1780. return;
  1781. /* Use the nominal value 10 ms if the read MDAT is zero,
  1782. * the nominal value of DETO is 20 ms.
  1783. */
  1784. if (dev->devslp_timing[ATA_LOG_DEVSLP_VALID] &
  1785. ATA_LOG_DEVSLP_VALID_MASK) {
  1786. mdat = dev->devslp_timing[ATA_LOG_DEVSLP_MDAT] &
  1787. ATA_LOG_DEVSLP_MDAT_MASK;
  1788. if (!mdat)
  1789. mdat = 10;
  1790. deto = dev->devslp_timing[ATA_LOG_DEVSLP_DETO];
  1791. if (!deto)
  1792. deto = 20;
  1793. } else {
  1794. mdat = 10;
  1795. deto = 20;
  1796. }
  1797. /* Make dito, mdat, deto bits to 0s */
  1798. devslp &= ~GENMASK_ULL(24, 2);
  1799. devslp |= ((dito << PORT_DEVSLP_DITO_OFFSET) |
  1800. (mdat << PORT_DEVSLP_MDAT_OFFSET) |
  1801. (deto << PORT_DEVSLP_DETO_OFFSET) |
  1802. PORT_DEVSLP_ADSE);
  1803. writel(devslp, port_mmio + PORT_DEVSLP);
  1804. hpriv->start_engine(ap);
  1805. /* enable device sleep feature for the drive */
  1806. err_mask = ata_dev_set_feature(dev,
  1807. SETFEATURES_SATA_ENABLE,
  1808. SATA_DEVSLP);
  1809. if (err_mask && err_mask != AC_ERR_DEV)
  1810. ata_dev_warn(dev, "failed to enable DEVSLP\n");
  1811. }
  1812. static void ahci_enable_fbs(struct ata_port *ap)
  1813. {
  1814. struct ahci_host_priv *hpriv = ap->host->private_data;
  1815. struct ahci_port_priv *pp = ap->private_data;
  1816. void __iomem *port_mmio = ahci_port_base(ap);
  1817. u32 fbs;
  1818. int rc;
  1819. if (!pp->fbs_supported)
  1820. return;
  1821. fbs = readl(port_mmio + PORT_FBS);
  1822. if (fbs & PORT_FBS_EN) {
  1823. pp->fbs_enabled = true;
  1824. pp->fbs_last_dev = -1; /* initialization */
  1825. return;
  1826. }
  1827. rc = hpriv->stop_engine(ap);
  1828. if (rc)
  1829. return;
  1830. writel(fbs | PORT_FBS_EN, port_mmio + PORT_FBS);
  1831. fbs = readl(port_mmio + PORT_FBS);
  1832. if (fbs & PORT_FBS_EN) {
  1833. dev_info(ap->host->dev, "FBS is enabled\n");
  1834. pp->fbs_enabled = true;
  1835. pp->fbs_last_dev = -1; /* initialization */
  1836. } else
  1837. dev_err(ap->host->dev, "Failed to enable FBS\n");
  1838. hpriv->start_engine(ap);
  1839. }
  1840. static void ahci_disable_fbs(struct ata_port *ap)
  1841. {
  1842. struct ahci_host_priv *hpriv = ap->host->private_data;
  1843. struct ahci_port_priv *pp = ap->private_data;
  1844. void __iomem *port_mmio = ahci_port_base(ap);
  1845. u32 fbs;
  1846. int rc;
  1847. if (!pp->fbs_supported)
  1848. return;
  1849. fbs = readl(port_mmio + PORT_FBS);
  1850. if ((fbs & PORT_FBS_EN) == 0) {
  1851. pp->fbs_enabled = false;
  1852. return;
  1853. }
  1854. rc = hpriv->stop_engine(ap);
  1855. if (rc)
  1856. return;
  1857. writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
  1858. fbs = readl(port_mmio + PORT_FBS);
  1859. if (fbs & PORT_FBS_EN)
  1860. dev_err(ap->host->dev, "Failed to disable FBS\n");
  1861. else {
  1862. dev_info(ap->host->dev, "FBS is disabled\n");
  1863. pp->fbs_enabled = false;
  1864. }
  1865. hpriv->start_engine(ap);
  1866. }
  1867. static void ahci_pmp_attach(struct ata_port *ap)
  1868. {
  1869. void __iomem *port_mmio = ahci_port_base(ap);
  1870. struct ahci_port_priv *pp = ap->private_data;
  1871. u32 cmd;
  1872. cmd = readl(port_mmio + PORT_CMD);
  1873. cmd |= PORT_CMD_PMP;
  1874. writel(cmd, port_mmio + PORT_CMD);
  1875. ahci_enable_fbs(ap);
  1876. pp->intr_mask |= PORT_IRQ_BAD_PMP;
  1877. /*
  1878. * We must not change the port interrupt mask register if the
  1879. * port is marked frozen, the value in pp->intr_mask will be
  1880. * restored later when the port is thawed.
  1881. *
  1882. * Note that during initialization, the port is marked as
  1883. * frozen since the irq handler is not yet registered.
  1884. */
  1885. if (!(ap->pflags & ATA_PFLAG_FROZEN))
  1886. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  1887. }
  1888. static void ahci_pmp_detach(struct ata_port *ap)
  1889. {
  1890. void __iomem *port_mmio = ahci_port_base(ap);
  1891. struct ahci_port_priv *pp = ap->private_data;
  1892. u32 cmd;
  1893. ahci_disable_fbs(ap);
  1894. cmd = readl(port_mmio + PORT_CMD);
  1895. cmd &= ~PORT_CMD_PMP;
  1896. writel(cmd, port_mmio + PORT_CMD);
  1897. pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
  1898. /* see comment above in ahci_pmp_attach() */
  1899. if (!(ap->pflags & ATA_PFLAG_FROZEN))
  1900. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  1901. }
  1902. int ahci_port_resume(struct ata_port *ap)
  1903. {
  1904. ahci_rpm_get_port(ap);
  1905. ahci_power_up(ap);
  1906. ahci_start_port(ap);
  1907. if (sata_pmp_attached(ap))
  1908. ahci_pmp_attach(ap);
  1909. else
  1910. ahci_pmp_detach(ap);
  1911. return 0;
  1912. }
  1913. EXPORT_SYMBOL_GPL(ahci_port_resume);
  1914. #ifdef CONFIG_PM
  1915. static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
  1916. {
  1917. const char *emsg = NULL;
  1918. int rc;
  1919. rc = ahci_deinit_port(ap, &emsg);
  1920. if (rc == 0)
  1921. ahci_power_down(ap);
  1922. else {
  1923. ata_port_err(ap, "%s (%d)\n", emsg, rc);
  1924. ata_port_freeze(ap);
  1925. }
  1926. ahci_rpm_put_port(ap);
  1927. return rc;
  1928. }
  1929. #endif
  1930. static int ahci_port_start(struct ata_port *ap)
  1931. {
  1932. struct ahci_host_priv *hpriv = ap->host->private_data;
  1933. struct device *dev = ap->host->dev;
  1934. struct ahci_port_priv *pp;
  1935. void *mem;
  1936. dma_addr_t mem_dma;
  1937. size_t dma_sz, rx_fis_sz;
  1938. pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
  1939. if (!pp)
  1940. return -ENOMEM;
  1941. if (ap->host->n_ports > 1) {
  1942. pp->irq_desc = devm_kzalloc(dev, 8, GFP_KERNEL);
  1943. if (!pp->irq_desc) {
  1944. devm_kfree(dev, pp);
  1945. return -ENOMEM;
  1946. }
  1947. snprintf(pp->irq_desc, 8,
  1948. "%s%d", dev_driver_string(dev), ap->port_no);
  1949. }
  1950. /* check FBS capability */
  1951. if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
  1952. void __iomem *port_mmio = ahci_port_base(ap);
  1953. u32 cmd = readl(port_mmio + PORT_CMD);
  1954. if (cmd & PORT_CMD_FBSCP)
  1955. pp->fbs_supported = true;
  1956. else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
  1957. dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
  1958. ap->port_no);
  1959. pp->fbs_supported = true;
  1960. } else
  1961. dev_warn(dev, "port %d is not capable of FBS\n",
  1962. ap->port_no);
  1963. }
  1964. if (pp->fbs_supported) {
  1965. dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
  1966. rx_fis_sz = AHCI_RX_FIS_SZ * 16;
  1967. } else {
  1968. dma_sz = AHCI_PORT_PRIV_DMA_SZ;
  1969. rx_fis_sz = AHCI_RX_FIS_SZ;
  1970. }
  1971. mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
  1972. if (!mem)
  1973. return -ENOMEM;
  1974. /*
  1975. * First item in chunk of DMA memory: 32-slot command table,
  1976. * 32 bytes each in size
  1977. */
  1978. pp->cmd_slot = mem;
  1979. pp->cmd_slot_dma = mem_dma;
  1980. mem += AHCI_CMD_SLOT_SZ;
  1981. mem_dma += AHCI_CMD_SLOT_SZ;
  1982. /*
  1983. * Second item: Received-FIS area
  1984. */
  1985. pp->rx_fis = mem;
  1986. pp->rx_fis_dma = mem_dma;
  1987. mem += rx_fis_sz;
  1988. mem_dma += rx_fis_sz;
  1989. /*
  1990. * Third item: data area for storing a single command
  1991. * and its scatter-gather table
  1992. */
  1993. pp->cmd_tbl = mem;
  1994. pp->cmd_tbl_dma = mem_dma;
  1995. /*
  1996. * Save off initial list of interrupts to be enabled.
  1997. * This could be changed later
  1998. */
  1999. pp->intr_mask = DEF_PORT_IRQ;
  2000. /*
  2001. * Switch to per-port locking in case each port has its own MSI vector.
  2002. */
  2003. if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
  2004. spin_lock_init(&pp->lock);
  2005. ap->lock = &pp->lock;
  2006. }
  2007. ap->private_data = pp;
  2008. /* engage engines, captain */
  2009. return ahci_port_resume(ap);
  2010. }
  2011. static void ahci_port_stop(struct ata_port *ap)
  2012. {
  2013. const char *emsg = NULL;
  2014. struct ahci_host_priv *hpriv = ap->host->private_data;
  2015. void __iomem *host_mmio = hpriv->mmio;
  2016. int rc;
  2017. /* de-initialize port */
  2018. rc = ahci_deinit_port(ap, &emsg);
  2019. if (rc)
  2020. ata_port_warn(ap, "%s (%d)\n", emsg, rc);
  2021. /*
  2022. * Clear GHC.IS to prevent stuck INTx after disabling MSI and
  2023. * re-enabling INTx.
  2024. */
  2025. writel(1 << ap->port_no, host_mmio + HOST_IRQ_STAT);
  2026. ahci_rpm_put_port(ap);
  2027. }
  2028. void ahci_print_info(struct ata_host *host, const char *scc_s)
  2029. {
  2030. struct ahci_host_priv *hpriv = host->private_data;
  2031. u32 vers, cap, cap2, impl, speed;
  2032. const char *speed_s;
  2033. vers = hpriv->version;
  2034. cap = hpriv->cap;
  2035. cap2 = hpriv->cap2;
  2036. impl = hpriv->port_map;
  2037. speed = (cap >> 20) & 0xf;
  2038. if (speed == 1)
  2039. speed_s = "1.5";
  2040. else if (speed == 2)
  2041. speed_s = "3";
  2042. else if (speed == 3)
  2043. speed_s = "6";
  2044. else
  2045. speed_s = "?";
  2046. dev_info(host->dev,
  2047. "AHCI %02x%02x.%02x%02x "
  2048. "%u slots %u ports %s Gbps 0x%x impl %s mode\n"
  2049. ,
  2050. (vers >> 24) & 0xff,
  2051. (vers >> 16) & 0xff,
  2052. (vers >> 8) & 0xff,
  2053. vers & 0xff,
  2054. ((cap >> 8) & 0x1f) + 1,
  2055. (cap & 0x1f) + 1,
  2056. speed_s,
  2057. impl,
  2058. scc_s);
  2059. dev_info(host->dev,
  2060. "flags: "
  2061. "%s%s%s%s%s%s%s"
  2062. "%s%s%s%s%s%s%s"
  2063. "%s%s%s%s%s%s%s"
  2064. "%s%s\n"
  2065. ,
  2066. cap & HOST_CAP_64 ? "64bit " : "",
  2067. cap & HOST_CAP_NCQ ? "ncq " : "",
  2068. cap & HOST_CAP_SNTF ? "sntf " : "",
  2069. cap & HOST_CAP_MPS ? "ilck " : "",
  2070. cap & HOST_CAP_SSS ? "stag " : "",
  2071. cap & HOST_CAP_ALPM ? "pm " : "",
  2072. cap & HOST_CAP_LED ? "led " : "",
  2073. cap & HOST_CAP_CLO ? "clo " : "",
  2074. cap & HOST_CAP_ONLY ? "only " : "",
  2075. cap & HOST_CAP_PMP ? "pmp " : "",
  2076. cap & HOST_CAP_FBS ? "fbs " : "",
  2077. cap & HOST_CAP_PIO_MULTI ? "pio " : "",
  2078. cap & HOST_CAP_SSC ? "slum " : "",
  2079. cap & HOST_CAP_PART ? "part " : "",
  2080. cap & HOST_CAP_CCC ? "ccc " : "",
  2081. cap & HOST_CAP_EMS ? "ems " : "",
  2082. cap & HOST_CAP_SXS ? "sxs " : "",
  2083. cap2 & HOST_CAP2_DESO ? "deso " : "",
  2084. cap2 & HOST_CAP2_SADM ? "sadm " : "",
  2085. cap2 & HOST_CAP2_SDS ? "sds " : "",
  2086. cap2 & HOST_CAP2_APST ? "apst " : "",
  2087. cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
  2088. cap2 & HOST_CAP2_BOH ? "boh " : ""
  2089. );
  2090. }
  2091. EXPORT_SYMBOL_GPL(ahci_print_info);
  2092. void ahci_set_em_messages(struct ahci_host_priv *hpriv,
  2093. struct ata_port_info *pi)
  2094. {
  2095. u8 messages;
  2096. void __iomem *mmio = hpriv->mmio;
  2097. u32 em_loc = readl(mmio + HOST_EM_LOC);
  2098. u32 em_ctl = readl(mmio + HOST_EM_CTL);
  2099. if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))
  2100. return;
  2101. messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
  2102. if (messages) {
  2103. /* store em_loc */
  2104. hpriv->em_loc = ((em_loc >> 16) * 4);
  2105. hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
  2106. hpriv->em_msg_type = messages;
  2107. pi->flags |= ATA_FLAG_EM;
  2108. if (!(em_ctl & EM_CTL_ALHD))
  2109. pi->flags |= ATA_FLAG_SW_ACTIVITY;
  2110. }
  2111. }
  2112. EXPORT_SYMBOL_GPL(ahci_set_em_messages);
  2113. static int ahci_host_activate_multi_irqs(struct ata_host *host,
  2114. struct scsi_host_template *sht)
  2115. {
  2116. struct ahci_host_priv *hpriv = host->private_data;
  2117. int i, rc;
  2118. rc = ata_host_start(host);
  2119. if (rc)
  2120. return rc;
  2121. /*
  2122. * Requests IRQs according to AHCI-1.1 when multiple MSIs were
  2123. * allocated. That is one MSI per port, starting from @irq.
  2124. */
  2125. for (i = 0; i < host->n_ports; i++) {
  2126. struct ahci_port_priv *pp = host->ports[i]->private_data;
  2127. int irq = hpriv->get_irq_vector(host, i);
  2128. /* Do not receive interrupts sent by dummy ports */
  2129. if (!pp) {
  2130. disable_irq(irq);
  2131. continue;
  2132. }
  2133. rc = devm_request_irq(host->dev, irq, ahci_multi_irqs_intr_hard,
  2134. 0, pp->irq_desc, host->ports[i]);
  2135. if (rc)
  2136. return rc;
  2137. ata_port_desc(host->ports[i], "irq %d", irq);
  2138. }
  2139. return ata_host_register(host, sht);
  2140. }
  2141. /**
  2142. * ahci_host_activate - start AHCI host, request IRQs and register it
  2143. * @host: target ATA host
  2144. * @sht: scsi_host_template to use when registering the host
  2145. *
  2146. * LOCKING:
  2147. * Inherited from calling layer (may sleep).
  2148. *
  2149. * RETURNS:
  2150. * 0 on success, -errno otherwise.
  2151. */
  2152. int ahci_host_activate(struct ata_host *host, struct scsi_host_template *sht)
  2153. {
  2154. struct ahci_host_priv *hpriv = host->private_data;
  2155. int irq = hpriv->irq;
  2156. int rc;
  2157. if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
  2158. if (hpriv->irq_handler &&
  2159. hpriv->irq_handler != ahci_single_level_irq_intr)
  2160. dev_warn(host->dev,
  2161. "both AHCI_HFLAG_MULTI_MSI flag set and custom irq handler implemented\n");
  2162. if (!hpriv->get_irq_vector) {
  2163. dev_err(host->dev,
  2164. "AHCI_HFLAG_MULTI_MSI requires ->get_irq_vector!\n");
  2165. return -EIO;
  2166. }
  2167. rc = ahci_host_activate_multi_irqs(host, sht);
  2168. } else {
  2169. rc = ata_host_activate(host, irq, hpriv->irq_handler,
  2170. IRQF_SHARED, sht);
  2171. }
  2172. return rc;
  2173. }
  2174. EXPORT_SYMBOL_GPL(ahci_host_activate);
  2175. MODULE_AUTHOR("Jeff Garzik");
  2176. MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
  2177. MODULE_LICENSE("GPL");