img_mem_man.c 69 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666
  1. /*!
  2. *****************************************************************************
  3. * Copyright (c) Imagination Technologies Ltd.
  4. *
  5. * The contents of this file are subject to the MIT license as set out below.
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation
  10. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11. * and/or sell copies of the Software, and to permit persons to whom the
  12. * Software is furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice shall be included in
  15. * all copies or substantial portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23. * THE SOFTWARE.
  24. *
  25. * Alternatively, the contents of this file may be used under the terms of the
  26. * GNU General Public License Version 2 ("GPL")in which case the provisions of
  27. * GPL are applicable instead of those above.
  28. *
  29. * If you wish to allow use of your version of this file only under the terms
  30. * of GPL, and not to allow others to use your version of this file under the
  31. * terms of the MIT license, indicate your decision by deleting the provisions
  32. * above and replace them with the notice and other provisions required by GPL
  33. * as set out in the file called "GPLHEADER" included in this distribution. If
  34. * you do not delete the provisions above, a recipient may use your version of
  35. * this file under the terms of either the MIT license or GPL.
  36. *
  37. * This License is also included in this distribution in the file called
  38. * "MIT_COPYING".
  39. *
  40. *****************************************************************************/
  41. #include <linux/module.h>
  42. #include <linux/init.h>
  43. #include <linux/mm.h>
  44. #include <linux/idr.h>
  45. #include <linux/mutex.h>
  46. #include <linux/list.h>
  47. #include <linux/slab.h>
  48. #include <linux/device.h>
  49. #include <linux/dma-mapping.h>
  50. #include <img_mem_man.h>
  51. #include <vha_drv_common.h>
  52. #include <mmu.h>
  53. #include <heap.h>
  54. #include "img_mem_man_priv.h"
  55. /* Maximum number of processes */
  56. #define MAX_PROC_CTX 1000
  57. /* Minimum page size (4KB) bits. */
  58. #define MIN_PAGE_SIZE_BITS 12
  59. struct mem_man {
  60. struct idr heaps;
  61. struct idr mem_ctxs;
  62. struct mutex mutex;
  63. unsigned cache_usage;
  64. };
  65. /* define like this, so it is easier to convert to a function argument later */
  66. static struct mem_man mem_man_data;
  67. /* wrapper struct for imgmmu_page */
  68. struct mmu_page {
  69. struct buffer *buffer;
  70. struct imgmmu_page page;
  71. unsigned char type;
  72. bool bypass_addr_trans;
  73. bool use_parity;
  74. };
  75. static bool trace_physical_pages;
  76. module_param(trace_physical_pages, bool, 0444);
  77. MODULE_PARM_DESC(trace_physical_pages,
  78. "Enables tracing of physical pages being mapped into MMU");
  79. static bool cache_sync = true;
  80. module_param(cache_sync, bool, 0444);
  81. MODULE_PARM_DESC(cache_sync,
  82. "cache sync mode: 0-no sync; 1-force sync (even if hw provides coherency);");
  83. /*
  84. * memory heaps
  85. */
  86. static char *get_heap_name(enum img_mem_heap_type type)
  87. {
  88. switch (type) {
  89. case IMG_MEM_HEAP_TYPE_UNIFIED:
  90. return "unified";
  91. case IMG_MEM_HEAP_TYPE_CARVEOUT:
  92. return "carveout";
  93. case IMG_MEM_HEAP_TYPE_ION:
  94. return "ion";
  95. case IMG_MEM_HEAP_TYPE_DMABUF:
  96. return "dmabuf";
  97. case IMG_MEM_HEAP_TYPE_COHERENT:
  98. return "coherent";
  99. case IMG_MEM_HEAP_TYPE_ANONYMOUS:
  100. return "anonymous";
  101. case IMG_MEM_HEAP_TYPE_OCM:
  102. return "ocm";
  103. default:
  104. WARN_ON(type);
  105. return "unknown";
  106. }
  107. }
  108. int img_mem_add_heap(const struct heap_config *heap_cfg, int *heap_id)
  109. {
  110. struct mem_man *mem_man = &mem_man_data;
  111. struct heap *heap;
  112. int (*init_fn)(const struct heap_config *heap_cfg, struct heap *heap);
  113. int ret;
  114. pr_debug("%s:%d\n", __func__, __LINE__);
  115. switch (heap_cfg->type) {
  116. case IMG_MEM_HEAP_TYPE_UNIFIED:
  117. init_fn = img_mem_unified_init;
  118. break;
  119. case IMG_MEM_HEAP_TYPE_COHERENT:
  120. init_fn = img_mem_coherent_init;
  121. break;
  122. #ifdef CONFIG_DMA_SHARED_BUFFER
  123. case IMG_MEM_HEAP_TYPE_DMABUF:
  124. init_fn = img_mem_dmabuf_init;
  125. break;
  126. #endif
  127. #ifdef ION_SUPPORTED
  128. #ifdef CONFIG_ION
  129. case IMG_MEM_HEAP_TYPE_ION:
  130. init_fn = img_mem_ion_init;
  131. break;
  132. #endif
  133. #endif
  134. #ifdef CONFIG_GENERIC_ALLOCATOR
  135. case IMG_MEM_HEAP_TYPE_CARVEOUT:
  136. init_fn = img_mem_carveout_init;
  137. break;
  138. #endif
  139. case IMG_MEM_HEAP_TYPE_ANONYMOUS:
  140. init_fn = img_mem_anonymous_init;
  141. break;
  142. case IMG_MEM_HEAP_TYPE_OCM:
  143. init_fn = img_mem_ocm_init;
  144. break;
  145. default:
  146. pr_err("%s: heap type %d unknown\n", __func__, heap_cfg->type);
  147. return -EINVAL;
  148. }
  149. heap = kmalloc(sizeof(struct heap), GFP_KERNEL);
  150. if (!heap)
  151. return -ENOMEM;
  152. ret = mutex_lock_interruptible(&mem_man->mutex);
  153. if (ret)
  154. goto lock_failed;
  155. ret = idr_alloc(&mem_man->heaps, heap, IMG_MEM_MAN_MIN_HEAP,
  156. IMG_MEM_MAN_MAX_HEAP, GFP_KERNEL);
  157. if (ret < 0) {
  158. pr_err("%s: idr_alloc failed\n", __func__);
  159. goto alloc_id_failed;
  160. }
  161. heap->id = ret;
  162. heap->type = heap_cfg->type;
  163. heap->options = heap_cfg->options;
  164. heap->to_dev_addr = heap_cfg->to_dev_addr;
  165. heap->to_host_addr = heap_cfg->to_host_addr;
  166. heap->priv = NULL;
  167. heap->cache_sync = true;
  168. heap->alt_cache_attr = heap_cfg->cache_attr;
  169. ret = init_fn(heap_cfg, heap);
  170. if (ret) {
  171. pr_err("%s: heap init failed\n", __func__);
  172. goto heap_init_failed;
  173. }
  174. *heap_id = heap->id;
  175. mutex_unlock(&mem_man->mutex);
  176. pr_debug("%s created heap %d type %d (%s)\n",
  177. __func__, *heap_id, heap_cfg->type, get_heap_name(heap->type));
  178. return 0;
  179. heap_init_failed:
  180. idr_remove(&mem_man->heaps, heap->id);
  181. alloc_id_failed:
  182. mutex_unlock(&mem_man->mutex);
  183. lock_failed:
  184. kfree(heap);
  185. return ret;
  186. }
  187. EXPORT_SYMBOL(img_mem_add_heap);
  188. static void _img_mem_del_heap(struct heap *heap)
  189. {
  190. struct mem_man *mem_man = &mem_man_data;
  191. pr_debug("%s heap %d 0x%p\n", __func__, heap->id, heap);
  192. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  193. if (heap->ops->destroy)
  194. heap->ops->destroy(heap);
  195. idr_remove(&mem_man->heaps, heap->id);
  196. }
  197. void img_mem_del_heap(int heap_id)
  198. {
  199. struct mem_man *mem_man = &mem_man_data;
  200. struct heap *heap;
  201. pr_debug("%s:%d heap %d\n", __func__, __LINE__, heap_id);
  202. mutex_lock(&mem_man->mutex);
  203. heap = idr_find(&mem_man->heaps, heap_id);
  204. if (!heap) {
  205. pr_warn("%s heap %d not found!\n", __func__, heap_id);
  206. mutex_unlock(&mem_man->mutex);
  207. return;
  208. }
  209. _img_mem_del_heap(heap);
  210. mutex_unlock(&mem_man->mutex);
  211. kfree(heap);
  212. }
  213. EXPORT_SYMBOL(img_mem_del_heap);
  214. int img_mem_get_heap_info(int heap_id, uint8_t *type, uint32_t *attrs)
  215. {
  216. struct mem_man *mem_man = &mem_man_data;
  217. struct heap *heap;
  218. pr_debug("%s:%d heap %d\n", __func__, __LINE__, heap_id);
  219. if (heap_id < IMG_MEM_MAN_MIN_HEAP || heap_id > IMG_MEM_MAN_MAX_HEAP) {
  220. pr_err("%s heap %d does not match internal constraints <%u - %u>!\n",
  221. __func__, heap_id, IMG_MEM_MAN_MIN_HEAP, IMG_MEM_MAN_MAX_HEAP);
  222. return -EINVAL;
  223. }
  224. mutex_lock(&mem_man->mutex);
  225. heap = idr_find(&mem_man->heaps, heap_id);
  226. if (!heap) {
  227. pr_debug("%s heap %d not found!\n", __func__, heap_id);
  228. mutex_unlock(&mem_man->mutex);
  229. return -ENOENT;
  230. }
  231. *type = heap->type;
  232. *attrs = 0;
  233. if (heap->ops->import)
  234. *attrs |= IMG_MEM_HEAP_ATTR_IMPORT;
  235. if (heap->ops->export)
  236. *attrs |= IMG_MEM_HEAP_ATTR_EXPORT;
  237. if (heap->ops->alloc && !heap->ops->import)
  238. *attrs |= IMG_MEM_HEAP_ATTR_INTERNAL;
  239. if (heap->type == IMG_MEM_HEAP_TYPE_OCM)
  240. *attrs = IMG_MEM_HEAP_ATTR_SEALED;
  241. /* User attributes */
  242. *attrs |= heap->options.ocm.hattr;
  243. mutex_unlock(&mem_man->mutex);
  244. return 0;
  245. }
  246. EXPORT_SYMBOL(img_mem_get_heap_info);
  247. /*
  248. * related to process context (contains SYSMEM heap's functionality in general)
  249. */
  250. int img_mem_create_proc_ctx(struct mem_ctx **new_ctx)
  251. {
  252. struct mem_man *mem_man = &mem_man_data;
  253. struct mem_ctx *ctx;
  254. int ret = 0;
  255. pr_debug("%s:%d\n", __func__, __LINE__);
  256. ctx = kzalloc(sizeof(struct mem_ctx), GFP_KERNEL);
  257. if (!ctx)
  258. return -ENOMEM;
  259. idr_init(&ctx->buffers);
  260. INIT_LIST_HEAD(&ctx->mmu_ctxs);
  261. mutex_lock(&mem_man->mutex);
  262. ret = idr_alloc(&mem_man->mem_ctxs, ctx, 0 , MAX_PROC_CTX,
  263. GFP_KERNEL);
  264. if (ret < 0) {
  265. mutex_unlock(&mem_man->mutex);
  266. pr_err("%s: idr_alloc failed\n", __func__);
  267. goto idr_alloc_failed;
  268. }
  269. /* Assign id to the newly created context. */
  270. ctx->id = ret;
  271. mutex_unlock(&mem_man->mutex);
  272. pr_debug("%s id:%d\n", __func__, ctx->id);
  273. *new_ctx = ctx;
  274. return 0;
  275. idr_alloc_failed:
  276. kfree(ctx);
  277. return ret;
  278. }
  279. EXPORT_SYMBOL(img_mem_create_proc_ctx);
  280. static void _img_mem_free(struct buffer *buffer);
  281. static void _img_mmu_unmap(struct mmu_ctx_mapping *mapping);
  282. static void _img_mmu_ctx_destroy(struct mmu_ctx *ctx);
  283. static void _img_mem_destroy_proc_ctx(struct mem_ctx *ctx)
  284. {
  285. struct mem_man *mem_man = &mem_man_data;
  286. struct buffer *buffer;
  287. int buf_id;
  288. pr_debug("%s:%d id:%d\n", __func__, __LINE__, ctx->id);
  289. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  290. /* free derelict mmu contexts */
  291. while (!list_empty(&ctx->mmu_ctxs)) {
  292. struct mmu_ctx *mc;
  293. mc = list_first_entry(&ctx->mmu_ctxs,
  294. struct mmu_ctx, mem_ctx_entry);
  295. pr_warn("%s: found derelict mmu context %p\n", __func__, mc);
  296. _img_mmu_ctx_destroy(mc);
  297. kfree(mc);
  298. }
  299. /* free derelict buffers */
  300. buf_id = IMG_MEM_MAN_MIN_BUFFER;
  301. buffer = idr_get_next(&ctx->buffers, &buf_id);
  302. while (buffer) {
  303. pr_warn("%s: found derelict buffer %d\n", __func__, buf_id);
  304. _img_mem_free(buffer);
  305. buf_id = IMG_MEM_MAN_MIN_BUFFER;
  306. buffer = idr_get_next(&ctx->buffers, &buf_id);
  307. }
  308. idr_destroy(&ctx->buffers);
  309. idr_remove(&mem_man->mem_ctxs, ctx->id);
  310. }
  311. void img_mem_destroy_proc_ctx(struct mem_ctx *ctx)
  312. {
  313. struct mem_man *mem_man = &mem_man_data;
  314. pr_debug("%s:%d\n", __func__, __LINE__);
  315. mutex_lock(&mem_man->mutex);
  316. _img_mem_destroy_proc_ctx(ctx);
  317. mutex_unlock(&mem_man->mutex);
  318. kfree(ctx);
  319. }
  320. EXPORT_SYMBOL(img_mem_destroy_proc_ctx);
  321. static int _img_mem_alloc(struct device *device, struct mem_ctx *ctx,
  322. struct heap *heap, size_t size,
  323. enum img_mem_attr attr, struct buffer **buffer_new)
  324. {
  325. struct mem_man *mem_man = &mem_man_data;
  326. struct buffer *buffer;
  327. int ret;
  328. /* Allocations for MMU pages are still 4k so CPU page size is enough */
  329. size_t align = attr & IMG_MEM_ATTR_MMU ?
  330. imgmmu_get_cpu_page_size() : IMGMMU_GET_MAX_PAGE_SIZE();
  331. pr_debug("%s heap %p '%s' ctx %p size %zu\n", __func__,
  332. heap, get_heap_name(heap->type), ctx, size);
  333. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  334. if (size == 0) {
  335. pr_err("%s: buffer size is zero\n", __func__);
  336. return -EINVAL;
  337. }
  338. if (heap->ops == NULL || heap->ops->alloc == NULL) {
  339. pr_err("%s: no alloc function in heap %d!\n",
  340. __func__, heap->id);
  341. return -EINVAL;
  342. }
  343. buffer = kzalloc(sizeof(struct buffer), GFP_KERNEL);
  344. if (!buffer)
  345. return -ENOMEM;
  346. ret = idr_alloc(&ctx->buffers, buffer,
  347. (IMG_MEM_MAN_MAX_BUFFER * ctx->id) +
  348. IMG_MEM_MAN_MIN_BUFFER,
  349. (IMG_MEM_MAN_MAX_BUFFER * ctx->id) +
  350. IMG_MEM_MAN_MAX_BUFFER,
  351. GFP_KERNEL);
  352. if (ret < 0) {
  353. pr_err("%s: idr_alloc failed\n", __func__);
  354. goto idr_alloc_failed;
  355. }
  356. buffer->id = ret;
  357. buffer->request_size = size;
  358. buffer->actual_size = ((size + align - 1) /
  359. align) * align;
  360. buffer->device = device;
  361. buffer->mem_ctx = ctx;
  362. buffer->heap = heap;
  363. INIT_LIST_HEAD(&buffer->mappings);
  364. buffer->kptr = NULL;
  365. buffer->priv = NULL;
  366. /* Check if heap has been registered using an alternative cache attributes */
  367. if (heap->alt_cache_attr &&
  368. (heap->alt_cache_attr != (attr & IMG_MEM_ATTR_CACHE_MASK))) {
  369. pr_debug("%s heap %d changing cache attributes from %x to %x\n",
  370. __func__, heap->id, attr & IMG_MEM_ATTR_CACHE_MASK,
  371. heap->alt_cache_attr);
  372. attr &= ~IMG_MEM_ATTR_CACHE_MASK;
  373. attr |= heap->alt_cache_attr;
  374. }
  375. ret = heap->ops->alloc(device, heap, buffer->actual_size, attr, buffer);
  376. if (ret) {
  377. pr_err("%s: heap %d alloc failed\n", __func__, heap->id);
  378. goto heap_alloc_failed;
  379. }
  380. if (heap->type != IMG_MEM_HEAP_TYPE_OCM) {
  381. __img_pdump_printf(device, "-- Allocating zeroed buffer id:%d size:%zu\n",
  382. buffer->id, buffer->actual_size);
  383. __img_pdump_printf(device, "CALLOC "_PMEM_":BLOCK_%d %#zx %#zx 0x0\n",
  384. buffer->id, buffer->actual_size, align);
  385. }
  386. ctx->mem_usage_curr += buffer->actual_size;
  387. if (ctx->mem_usage_curr > ctx->mem_usage_max)
  388. ctx->mem_usage_max = ctx->mem_usage_curr;
  389. *buffer_new = buffer;
  390. pr_debug("%s heap %p ctx %p created buffer %d (%p) actual_size %zu\n",
  391. __func__, heap, ctx, buffer->id, buffer, buffer->actual_size);
  392. return 0;
  393. heap_alloc_failed:
  394. idr_remove(&ctx->buffers, buffer->id);
  395. idr_alloc_failed:
  396. kfree(buffer);
  397. return ret;
  398. }
  399. int img_mem_alloc(struct device *device, struct mem_ctx *ctx, int heap_id,
  400. size_t size, enum img_mem_attr attr, int *buf_id)
  401. {
  402. struct mem_man *mem_man = &mem_man_data;
  403. struct heap *heap;
  404. struct buffer *buffer;
  405. int ret;
  406. pr_debug("%s heap %d ctx %p size %zu\n", __func__, heap_id, ctx, size);
  407. ret = mutex_lock_interruptible(&mem_man->mutex);
  408. if (ret)
  409. return ret;
  410. heap = idr_find(&mem_man->heaps, heap_id);
  411. if (!heap) {
  412. pr_err("%s: heap id %d not found\n", __func__, heap_id);
  413. mutex_unlock(&mem_man->mutex);
  414. return -EINVAL;
  415. }
  416. ret = _img_mem_alloc(device, ctx, heap, size, attr, &buffer);
  417. if (ret) {
  418. mutex_unlock(&mem_man->mutex);
  419. return ret;
  420. }
  421. *buf_id = buffer->id;
  422. mutex_unlock(&mem_man->mutex);
  423. pr_debug("%s heap %d ctx %p created buffer %d (%p) size %zu\n",
  424. __func__, heap_id, ctx, *buf_id, buffer, size);
  425. return ret;
  426. }
  427. EXPORT_SYMBOL(img_mem_alloc);
  428. static int _img_mem_import(struct device *device,
  429. struct mem_ctx *ctx, struct heap *heap,
  430. size_t size, enum img_mem_attr attr, uint64_t buf_hnd,
  431. struct buffer **buffer_new)
  432. {
  433. struct mem_man *mem_man = &mem_man_data;
  434. struct buffer *buffer;
  435. int ret;
  436. size_t align = IMGMMU_GET_MAX_PAGE_SIZE();
  437. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  438. if (size == 0) {
  439. pr_err("%s: buffer size is zero\n", __func__);
  440. return -EINVAL;
  441. }
  442. if (heap->ops == NULL || heap->ops->import == NULL) {
  443. pr_err("%s: no import function in heap %d!\n",
  444. __func__, heap->id);
  445. return -EINVAL;
  446. }
  447. buffer = kzalloc(sizeof(struct buffer), GFP_KERNEL);
  448. if (!buffer)
  449. return -ENOMEM;
  450. ret = idr_alloc(&ctx->buffers, buffer,
  451. (IMG_MEM_MAN_MAX_BUFFER * ctx->id) +
  452. IMG_MEM_MAN_MIN_BUFFER,
  453. (IMG_MEM_MAN_MAX_BUFFER * ctx->id) +
  454. IMG_MEM_MAN_MAX_BUFFER,
  455. GFP_KERNEL);
  456. if (ret < 0) {
  457. pr_err("%s: idr_alloc failed\n", __func__);
  458. goto idr_alloc_failed;
  459. }
  460. buffer->id = ret;
  461. buffer->request_size = size;
  462. buffer->actual_size = ((size + align - 1) /
  463. align) * align;
  464. buffer->device = device;
  465. buffer->mem_ctx = ctx;
  466. buffer->heap = heap;
  467. INIT_LIST_HEAD(&buffer->mappings);
  468. buffer->kptr = NULL;
  469. buffer->priv = NULL;
  470. /* If MMU page size is bigger than CPU page size
  471. * we need an extra check against requested size
  472. * The aligned size comparing to requested size
  473. * can't be bigger than CPU page!
  474. * otherwise it can cause troubles when
  475. * HW tries to access non existing pages */
  476. if (buffer->actual_size - buffer->request_size >
  477. imgmmu_get_cpu_page_size()) {
  478. pr_err("%s: original buffer size is not MMU page size aligned!\n",
  479. __func__);
  480. ret = -EINVAL;
  481. goto heap_import_failed;
  482. }
  483. /* Check if heap has been registered using an alternative cache attributes */
  484. if (heap->alt_cache_attr &&
  485. (heap->alt_cache_attr != (attr & IMG_MEM_ATTR_CACHE_MASK))) {
  486. pr_debug("%s heap %d changing cache attributes from %x to %x\n",
  487. __func__, heap->id, attr & IMG_MEM_ATTR_CACHE_MASK,
  488. heap->alt_cache_attr);
  489. attr &= ~IMG_MEM_ATTR_CACHE_MASK;
  490. attr |= heap->alt_cache_attr;
  491. }
  492. ret = heap->ops->import(device, heap, buffer->actual_size, attr,
  493. buf_hnd, buffer);
  494. if (ret) {
  495. pr_err("%s: heap %d import failed\n", __func__, heap->id);
  496. goto heap_import_failed;
  497. }
  498. __img_pdump_printf(device, "-- Allocating zeroed buffer id:%d size:%zu for imported data\n",
  499. buffer->id, buffer->actual_size);
  500. __img_pdump_printf(device, "CALLOC "_PMEM_":BLOCK_%d %#zx %#zx 0x0\n",
  501. buffer->id, buffer->actual_size, align);
  502. ctx->mem_usage_curr += buffer->actual_size;
  503. if (ctx->mem_usage_curr > ctx->mem_usage_max)
  504. ctx->mem_usage_max = ctx->mem_usage_curr;
  505. *buffer_new = buffer;
  506. return 0;
  507. heap_import_failed:
  508. idr_remove(&ctx->buffers, buffer->id);
  509. idr_alloc_failed:
  510. kfree(buffer);
  511. return ret;
  512. }
  513. int img_mem_import(struct device *device, struct mem_ctx *ctx, int heap_id,
  514. size_t size, enum img_mem_attr attr, uint64_t buf_hnd,
  515. int *buf_id)
  516. {
  517. struct mem_man *mem_man = &mem_man_data;
  518. struct heap *heap;
  519. struct buffer *buffer;
  520. int ret;
  521. pr_debug("%s heap %d ctx %p hnd %#llx\n", __func__, heap_id, ctx, buf_hnd);
  522. ret = mutex_lock_interruptible(&mem_man->mutex);
  523. if (ret)
  524. return ret;
  525. heap = idr_find(&mem_man->heaps, heap_id);
  526. if (!heap) {
  527. pr_err("%s: heap id %d not found\n", __func__, heap_id);
  528. mutex_unlock(&mem_man->mutex);
  529. return -EINVAL;
  530. }
  531. ret = _img_mem_import(device, ctx, heap, size, attr, buf_hnd, &buffer);
  532. if (ret) {
  533. mutex_unlock(&mem_man->mutex);
  534. return ret;
  535. }
  536. *buf_id = buffer->id;
  537. mutex_unlock(&mem_man->mutex);
  538. pr_info("%s buf_hnd %#llx heap %d (%s) buffer %d size %zu\n", __func__,
  539. buf_hnd, heap_id, get_heap_name(heap->type), *buf_id, size);
  540. pr_debug("%s heap %d ctx %p created buffer %d (%p) size %zu\n",
  541. __func__, heap_id, ctx, *buf_id, buffer, size);
  542. return ret;
  543. }
  544. EXPORT_SYMBOL(img_mem_import);
  545. static int _img_mem_export(struct device *device,
  546. struct mem_ctx *ctx, struct heap *heap,
  547. size_t size, enum img_mem_attr attr,
  548. struct buffer *buffer, uint64_t *buf_hnd)
  549. {
  550. struct mem_man *mem_man = &mem_man_data;
  551. int ret;
  552. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  553. if (size > buffer->actual_size) {
  554. pr_err("%s: buffer size (%zu) bigger than actual size (%zu)\n",
  555. __func__, size, buffer->actual_size);
  556. return -EINVAL;
  557. }
  558. if (heap->ops == NULL || heap->ops->export == NULL) {
  559. pr_err("%s: no export function in heap %d!\n",
  560. __func__, heap->id);
  561. return -EINVAL;
  562. }
  563. ret = heap->ops->export(device, heap, buffer->actual_size, attr,
  564. buffer, buf_hnd);
  565. if (ret) {
  566. pr_err("%s: heap %d export failed\n", __func__, heap->id);
  567. return -EFAULT;
  568. }
  569. return ret;
  570. }
  571. int img_mem_export(struct device *device, struct mem_ctx *ctx, int buf_id,
  572. size_t size, enum img_mem_attr attr, uint64_t *buf_hnd)
  573. {
  574. struct mem_man *mem_man = &mem_man_data;
  575. struct heap *heap;
  576. struct buffer *buffer;
  577. int ret;
  578. pr_debug("%s ctx %p buffer id %d\n", __func__, ctx, buf_id);
  579. ret = mutex_lock_interruptible(&mem_man->mutex);
  580. if (ret)
  581. return ret;
  582. buffer = idr_find(&ctx->buffers, buf_id);
  583. if (!buffer) {
  584. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  585. mutex_unlock(&mem_man->mutex);
  586. return -EINVAL;
  587. }
  588. heap = buffer->heap;
  589. ret = _img_mem_export(device, ctx, heap, size, attr, buffer, buf_hnd);
  590. if (ret) {
  591. mutex_unlock(&mem_man->mutex);
  592. return ret;
  593. }
  594. mutex_unlock(&mem_man->mutex);
  595. pr_info("%s buf_hnd %#llx heap %d (%s) buffer %d size %zu\n", __func__,
  596. *buf_hnd, heap->id, get_heap_name(heap->type), buf_id, size);
  597. pr_debug("%s heap %d ctx %p exported buffer %d (%p) size %zu\n",
  598. __func__, heap->id, ctx, buf_id, buffer, size);
  599. return ret;
  600. }
  601. EXPORT_SYMBOL(img_mem_export);
  602. static void _img_mem_free(struct buffer *buffer)
  603. {
  604. struct mem_man *mem_man = &mem_man_data;
  605. struct heap *heap = buffer->heap;
  606. struct mem_ctx *ctx = buffer->mem_ctx;
  607. pr_debug("%s buffer 0x%p\n", __func__, buffer);
  608. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  609. if (heap->ops == NULL || heap->ops->free == NULL) {
  610. pr_err("%s: no free function in heap %d!\n",
  611. __func__, heap->id);
  612. return;
  613. }
  614. while (!list_empty(&buffer->mappings)) {
  615. struct mmu_ctx_mapping *map;
  616. map = list_first_entry(&buffer->mappings,
  617. struct mmu_ctx_mapping, buffer_entry);
  618. pr_debug("%s: found mapping for buffer %d (size %zu)\n",
  619. __func__, map->buffer->id, map->buffer->actual_size);
  620. _img_mmu_unmap(map);
  621. kfree(map);
  622. }
  623. heap->ops->free(heap, buffer);
  624. if (ctx->mem_usage_curr >= buffer->actual_size)
  625. ctx->mem_usage_curr -= buffer->actual_size;
  626. else
  627. WARN_ON(1);
  628. idr_remove(&ctx->buffers, buffer->id);
  629. if (heap->type != IMG_MEM_HEAP_TYPE_OCM) {
  630. __img_pdump_printf(buffer->device, "-- Freeing buffer id:%d size:%zu\n",
  631. buffer->id, buffer->actual_size);
  632. __img_pdump_printf(buffer->device, "FREE "_PMEM_":BLOCK_%d\n", buffer->id);
  633. }
  634. kfree(buffer);
  635. }
  636. void img_mem_free(struct mem_ctx *ctx, int buf_id)
  637. {
  638. struct mem_man *mem_man = &mem_man_data;
  639. struct buffer *buffer;
  640. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  641. mutex_lock(&mem_man->mutex);
  642. buffer = idr_find(&ctx->buffers, buf_id);
  643. if (!buffer) {
  644. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  645. mutex_unlock(&mem_man->mutex);
  646. return;
  647. }
  648. _img_mem_free(buffer);
  649. mutex_unlock(&mem_man->mutex);
  650. }
  651. EXPORT_SYMBOL(img_mem_free);
  652. #ifdef KERNEL_DMA_FENCE_SUPPORT
  653. /*
  654. * dma_fence ops
  655. */
  656. static const char *_img_mem_sync_get_driver_name(struct dma_fence *f)
  657. {
  658. return "buf_sync";
  659. }
  660. static const char *_img_mem_sync_get_timeline_name(struct dma_fence *f)
  661. {
  662. return "buf_timeline";
  663. }
  664. static bool _img_mem_sync_enable_signaling(struct dma_fence *f)
  665. {
  666. return true;
  667. }
  668. static void _img_mem_sync_release(struct dma_fence *fence)
  669. {
  670. dma_fence_free(fence);
  671. }
  672. static struct dma_fence_ops dma_fence_ops = {
  673. .get_driver_name = _img_mem_sync_get_driver_name,
  674. .get_timeline_name = _img_mem_sync_get_timeline_name,
  675. .enable_signaling = _img_mem_sync_enable_signaling,
  676. .release = _img_mem_sync_release,
  677. .wait = dma_fence_default_wait
  678. };
  679. struct dma_fence * img_mem_add_fence(struct mem_ctx *ctx, int buf_id)
  680. {
  681. struct mem_man *mem_man = &mem_man_data;
  682. struct buffer *buffer;
  683. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  684. mutex_lock(&mem_man->mutex);
  685. buffer = idr_find(&ctx->buffers, buf_id);
  686. if (!buffer) {
  687. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  688. mutex_unlock(&mem_man->mutex);
  689. return NULL;
  690. }
  691. if (buffer->fence) {
  692. pr_err("%s: fence for buffer id %d already allocated and not freed \n",
  693. __func__, buf_id);
  694. mutex_unlock(&mem_man->mutex);
  695. return NULL;
  696. }
  697. buffer->fence = kmalloc(sizeof(struct buffer_fence), GFP_KERNEL);
  698. if (!buffer->fence) {
  699. pr_err("%s: cannot allocate fence for buffer id %d\n", __func__, buf_id);
  700. mutex_unlock(&mem_man->mutex);
  701. return NULL;
  702. }
  703. spin_lock_init(&buffer->fence->lock);
  704. dma_fence_init(&buffer->fence->fence,
  705. &dma_fence_ops,
  706. &buffer->fence->lock,
  707. dma_fence_context_alloc(1),
  708. 1);
  709. mutex_unlock(&mem_man->mutex);
  710. return &buffer->fence->fence;
  711. }
  712. EXPORT_SYMBOL(img_mem_add_fence);
  713. void img_mem_remove_fence(struct mem_ctx *ctx, int buf_id)
  714. {
  715. struct mem_man *mem_man = &mem_man_data;
  716. struct buffer *buffer;
  717. struct dma_fence *fence = NULL;
  718. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  719. mutex_lock(&mem_man->mutex);
  720. buffer = idr_find(&ctx->buffers, buf_id);
  721. if (!buffer) {
  722. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  723. mutex_unlock(&mem_man->mutex);
  724. return;
  725. }
  726. if (buffer->fence) {
  727. fence = &buffer->fence->fence;
  728. buffer->fence = NULL;
  729. }
  730. mutex_unlock(&mem_man->mutex);
  731. if (fence)
  732. dma_fence_signal(fence);
  733. }
  734. EXPORT_SYMBOL(img_mem_remove_fence);
  735. int img_mem_signal_fence(struct mem_ctx *ctx, int buf_id)
  736. {
  737. struct mem_man *mem_man = &mem_man_data;
  738. struct buffer *buffer;
  739. struct dma_fence *fence = NULL;
  740. int ret = -1;
  741. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  742. mutex_lock(&mem_man->mutex);
  743. buffer = idr_find(&ctx->buffers, buf_id);
  744. if (!buffer) {
  745. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  746. mutex_unlock(&mem_man->mutex);
  747. return -1;
  748. }
  749. if (buffer->fence) {
  750. fence = &buffer->fence->fence;
  751. buffer->fence = NULL;
  752. }
  753. mutex_unlock(&mem_man->mutex);
  754. if (fence)
  755. ret = dma_fence_signal(fence);
  756. return ret;
  757. }
  758. EXPORT_SYMBOL(img_mem_signal_fence);
  759. #endif
  760. static void _img_mem_sync_device_to_cpu(struct buffer *buffer, bool force);
  761. int img_mem_map_um(struct mem_ctx *ctx, int buf_id, struct vm_area_struct *vma)
  762. {
  763. struct mem_man *mem_man = &mem_man_data;
  764. struct buffer *buffer;
  765. struct heap *heap;
  766. int ret;
  767. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  768. mutex_lock(&mem_man->mutex);
  769. buffer = idr_find(&ctx->buffers, buf_id);
  770. if (!buffer) {
  771. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  772. mutex_unlock(&mem_man->mutex);
  773. return -EINVAL;
  774. }
  775. pr_debug("%s:%d buffer 0x%p\n", __func__, __LINE__, buffer);
  776. heap = buffer->heap;
  777. if (heap->ops == NULL || heap->ops->map_um == NULL) {
  778. pr_err("%s: no map_um in heap %d!\n", __func__, heap->id);
  779. mutex_unlock(&mem_man->mutex);
  780. return -EINVAL;
  781. }
  782. ret = heap->ops->map_um(heap, buffer, vma);
  783. /* Always invalidate the buffer when it is mapped into UM for reading */
  784. if (!ret && (vma->vm_flags & VM_READ) && !(vma->vm_flags & VM_WRITE))
  785. _img_mem_sync_device_to_cpu(buffer, false);
  786. mutex_unlock(&mem_man->mutex);
  787. return ret;
  788. }
  789. EXPORT_SYMBOL(img_mem_map_um);
  790. int img_mem_unmap_um(struct mem_ctx *ctx, int buf_id)
  791. {
  792. struct mem_man *mem_man = &mem_man_data;
  793. struct buffer *buffer;
  794. struct heap *heap;
  795. int ret;
  796. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  797. mutex_lock(&mem_man->mutex);
  798. buffer = idr_find(&ctx->buffers, buf_id);
  799. if (!buffer) {
  800. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  801. mutex_unlock(&mem_man->mutex);
  802. return -EINVAL;
  803. }
  804. pr_debug("%s:%d buffer 0x%p\n", __func__, __LINE__, buffer);
  805. heap = buffer->heap;
  806. if (heap->ops == NULL || heap->ops->unmap_um == NULL) {
  807. pr_err("%s: no map_um in heap %d!\n", __func__, heap->id);
  808. mutex_unlock(&mem_man->mutex);
  809. return -EINVAL;
  810. }
  811. ret = heap->ops->unmap_um(heap, buffer);
  812. mutex_unlock(&mem_man->mutex);
  813. return ret;
  814. }
  815. EXPORT_SYMBOL(img_mem_unmap_um);
  816. static int _img_mem_map_km(struct buffer *buffer)
  817. {
  818. struct mem_man *mem_man = &mem_man_data;
  819. struct heap *heap = buffer->heap;
  820. pr_debug("%s:%d buffer 0x%p\n", __func__, __LINE__, buffer);
  821. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  822. if (heap->ops == NULL || heap->ops->map_km == NULL) {
  823. pr_err("%s: no map_km in heap %d!\n", __func__, heap->id);
  824. return -EINVAL;
  825. }
  826. return heap->ops->map_km(heap, buffer);
  827. }
  828. int img_mem_map_km(struct mem_ctx *ctx, int buf_id)
  829. {
  830. struct mem_man *mem_man = &mem_man_data;
  831. struct buffer *buffer;
  832. int ret;
  833. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  834. mutex_lock(&mem_man->mutex);
  835. buffer = idr_find(&ctx->buffers, buf_id);
  836. if (!buffer) {
  837. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  838. mutex_unlock(&mem_man->mutex);
  839. return -EINVAL;
  840. }
  841. ret = _img_mem_map_km(buffer);
  842. mutex_unlock(&mem_man->mutex);
  843. return ret;
  844. }
  845. EXPORT_SYMBOL(img_mem_map_km);
  846. static int _img_mem_unmap_km(struct buffer *buffer)
  847. {
  848. struct mem_man *mem_man = &mem_man_data;
  849. struct heap *heap = buffer->heap;
  850. pr_debug("%s:%d buffer 0x%p\n", __func__, __LINE__, buffer);
  851. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  852. if (heap->ops == NULL || heap->ops->unmap_km == NULL) {
  853. pr_err("%s: no unmap_km in heap %d!\n", __func__, heap->id);
  854. return -EINVAL;
  855. }
  856. return heap->ops->unmap_km(heap, buffer);
  857. }
  858. int img_mem_unmap_km(struct mem_ctx *ctx, int buf_id)
  859. {
  860. struct mem_man *mem_man = &mem_man_data;
  861. struct buffer *buffer;
  862. int ret;
  863. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  864. mutex_lock(&mem_man->mutex);
  865. buffer = idr_find(&ctx->buffers, buf_id);
  866. if (!buffer) {
  867. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  868. mutex_unlock(&mem_man->mutex);
  869. return -EINVAL;
  870. }
  871. ret = _img_mem_unmap_km(buffer);
  872. mutex_unlock(&mem_man->mutex);
  873. return ret;
  874. }
  875. EXPORT_SYMBOL(img_mem_unmap_km);
  876. uint64_t *img_mem_get_page_array(struct mem_ctx *mem_ctx, int buf_id)
  877. {
  878. struct buffer *buffer;
  879. struct heap *heap;
  880. struct mem_man *mem_man = &mem_man_data;
  881. uint64_t *addrs = NULL;
  882. int ret;
  883. mutex_lock(&mem_man->mutex);
  884. buffer = idr_find(&mem_ctx->buffers, buf_id);
  885. if (!buffer) {
  886. pr_err("%s: buffer id %d not found\n",
  887. __func__, buf_id);
  888. mutex_unlock(&mem_man->mutex);
  889. return NULL;
  890. }
  891. heap = buffer->heap;
  892. if (heap && heap->ops && heap->ops->get_page_array) {
  893. ret = heap->ops->get_page_array(heap, buffer, &addrs);
  894. if (ret || addrs == NULL) {
  895. pr_err("%s: no page array for heap %d buffer %d\n",
  896. __func__, heap->id, buffer->id);
  897. }
  898. } else
  899. pr_err("%s: heap %d does not support page arrays\n",
  900. __func__, heap->id);
  901. mutex_unlock(&mem_man->mutex);
  902. return addrs;
  903. }
  904. EXPORT_SYMBOL(img_mem_get_page_array);
  905. /* gets physical address of a single page at given offset */
  906. uint64_t img_mem_get_single_page(struct mem_ctx *mem_ctx, int buf_id,
  907. unsigned int offset)
  908. {
  909. struct buffer *buffer;
  910. struct heap *heap;
  911. struct mem_man *mem_man = &mem_man_data;
  912. int ret;
  913. uint64_t addr = 0;
  914. mutex_lock(&mem_man->mutex);
  915. buffer = idr_find(&mem_ctx->buffers, buf_id);
  916. if (!buffer) {
  917. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  918. mutex_unlock(&mem_man->mutex);
  919. return -1;
  920. }
  921. heap = buffer->heap;
  922. if (!heap) {
  923. pr_err("%s: buffer %d does not point any heap it belongs to!\n",
  924. __func__, buf_id);
  925. mutex_unlock(&mem_man->mutex);
  926. return -1;
  927. }
  928. if (heap->ops && heap->ops->get_sg_table) {
  929. struct sg_table *sgt;
  930. struct scatterlist *sgl;
  931. int offs = offset;
  932. bool use_sg_dma = false;
  933. ret = heap->ops->get_sg_table(heap, buffer, &sgt, &use_sg_dma);
  934. if (ret) {
  935. pr_err("%s: heap %d buffer %d no sg_table!\n",
  936. __func__, heap->id, buffer->id);
  937. return -1;
  938. }
  939. sgl = sgt->sgl;
  940. while (sgl) {
  941. if (use_sg_dma)
  942. offs -= sg_dma_len(sgl);
  943. else
  944. offs -= sgl->length;
  945. if (offs <= 0)
  946. break;
  947. sgl = sg_next(sgl);
  948. }
  949. if (!sgl) {
  950. pr_err("%s: heap %d buffer %d wrong offset %d!\n",
  951. __func__, heap->id, buffer->id, offset);
  952. return -1;
  953. }
  954. if (use_sg_dma)
  955. addr = sg_dma_address(sgl);
  956. else
  957. addr = sg_phys(sgl);
  958. } else if (heap->ops && heap->ops->get_page_array) {
  959. uint64_t *addrs;
  960. int page_idx = offset / PAGE_SIZE;
  961. ret = heap->ops->get_page_array(heap, buffer, &addrs);
  962. if (ret) {
  963. pr_err("%s: heap %d buffer %d no page array!\n",
  964. __func__, heap->id, buffer->id);
  965. return -1;
  966. }
  967. if (offset > buffer->actual_size) {
  968. pr_err("%s: heap %d buffer %d wrong offset %d!\n",
  969. __func__, heap->id, buffer->id, offset);
  970. return -1;
  971. }
  972. addr = addrs[page_idx];
  973. }
  974. mutex_unlock(&mem_man->mutex);
  975. return addr;
  976. }
  977. EXPORT_SYMBOL(img_mem_get_single_page);
  978. void *img_mem_get_kptr(struct mem_ctx *ctx, int buf_id)
  979. {
  980. struct mem_man *mem_man = &mem_man_data;
  981. struct buffer *buffer;
  982. void *kptr;
  983. mutex_lock(&mem_man->mutex);
  984. buffer = idr_find(&ctx->buffers, buf_id);
  985. if (!buffer) {
  986. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  987. mutex_unlock(&mem_man->mutex);
  988. return NULL;
  989. }
  990. kptr = buffer->kptr;
  991. mutex_unlock(&mem_man->mutex);
  992. return kptr;
  993. }
  994. EXPORT_SYMBOL(img_mem_get_kptr);
  995. phys_addr_t img_mem_get_dev_addr(struct mem_ctx *mem_ctx, int buf_id,
  996. phys_addr_t addr)
  997. {
  998. struct mem_man *mem_man = &mem_man_data;
  999. struct buffer *buffer;
  1000. struct heap *heap;
  1001. mutex_lock(&mem_man->mutex);
  1002. buffer = idr_find(&mem_ctx->buffers, buf_id);
  1003. if (!buffer) {
  1004. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  1005. mutex_unlock(&mem_man->mutex);
  1006. return addr;
  1007. }
  1008. heap = buffer->heap;
  1009. if (heap->to_dev_addr)
  1010. addr = heap->to_dev_addr(&heap->options, addr);
  1011. mutex_unlock(&mem_man->mutex);
  1012. return addr;
  1013. }
  1014. EXPORT_SYMBOL(img_mem_get_dev_addr);
  1015. int img_mmu_init_cache(struct mmu_ctx *mmu_ctx, unsigned long cache_phys_start,
  1016. uint32_t cache_size)
  1017. {
  1018. struct mem_man *mem_man = &mem_man_data;
  1019. struct pdump_descr* pdump = vha_pdump_dev_get_drvdata(mmu_ctx->device);
  1020. mutex_lock(&mem_man->mutex);
  1021. mmu_ctx->cache_phys_start = cache_phys_start;
  1022. mmu_ctx->cache_size = cache_size;
  1023. if (img_pdump_enabled(pdump) && cache_size && !mem_man->cache_usage) {
  1024. __img_pdump_printf(mmu_ctx->device, "-- Allocating img mem cache buffer size:%u\n", cache_size);
  1025. __img_pdump_printf(mmu_ctx->device, "CALLOC :OCM:BLOCK_CACHE %#x %#zx 0x0\n",
  1026. cache_size, IMGMMU_GET_MAX_PAGE_SIZE());
  1027. }
  1028. mem_man->cache_usage++;
  1029. mutex_unlock(&mem_man->mutex);
  1030. return 0;
  1031. }
  1032. EXPORT_SYMBOL(img_mmu_init_cache);
  1033. int img_mmu_clear_cache(struct mmu_ctx *mmu_ctx)
  1034. {
  1035. struct mem_man *mem_man = &mem_man_data;
  1036. struct pdump_descr* pdump = vha_pdump_dev_get_drvdata(mmu_ctx->device);
  1037. mutex_lock(&mem_man->mutex);
  1038. if (mem_man->cache_usage)
  1039. mem_man->cache_usage--;
  1040. if (img_pdump_enabled(pdump) && mmu_ctx->cache_size && !mem_man->cache_usage) {
  1041. __img_pdump_printf(mmu_ctx->device, "-- Freeing img mem cache buffer size:%u\n",
  1042. mmu_ctx->cache_size);
  1043. __img_pdump_printf(mmu_ctx->device, "FREE :OCM:BLOCK_CACHE\n");
  1044. }
  1045. mutex_unlock(&mem_man->mutex);
  1046. return 0;
  1047. }
  1048. EXPORT_SYMBOL(img_mmu_clear_cache);
  1049. int img_mmu_move_pg_to_cache(struct mmu_ctx *mmu_ctx, struct mem_ctx *mem_ctx,
  1050. int buf_id, uint64_t virt_addr, uint32_t page_size, uint32_t page_idx)
  1051. {
  1052. struct mem_man *mem_man = &mem_man_data;
  1053. struct buffer *buffer;
  1054. struct mmu_ctx_mapping *mapping;
  1055. int ret = -EINVAL;
  1056. if (page_size != imgmmu_get_page_size()) {
  1057. pr_err("%s: page sizes does not match!\n", __func__);
  1058. return -EINVAL;
  1059. }
  1060. if (!mmu_ctx->mmu_cat) {
  1061. pr_err("%s: trying to move pages with mmu disabled!\n", __func__);
  1062. return -EINVAL;
  1063. }
  1064. mutex_lock(&mem_man->mutex);
  1065. buffer = idr_find(&mem_ctx->buffers, buf_id);
  1066. if (!buffer) {
  1067. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  1068. mutex_unlock(&mem_man->mutex);
  1069. return -EINVAL;
  1070. }
  1071. if (buffer->actual_size <= page_idx * imgmmu_get_page_size()) {
  1072. pr_err("%s: trying to remap out of the buffer boundaries!\n", __func__);
  1073. mutex_unlock(&mem_man->mutex);
  1074. return -EINVAL;
  1075. }
  1076. list_for_each_entry(mapping, &buffer->mappings, buffer_entry) {
  1077. if (mapping->virt_addr == virt_addr) {
  1078. if (mapping->cache_offset + imgmmu_get_page_size() <= mmu_ctx->cache_size) {
  1079. __img_pdump_printf(buffer->device, "-- Move page to CACHE\n");
  1080. ret = imgmmu_cat_override_phys_addr(mmu_ctx->mmu_cat,
  1081. mapping->virt_addr + page_idx * imgmmu_get_page_size(),
  1082. mmu_ctx->cache_phys_start + mapping->cache_offset);
  1083. mapping->cache_offset += imgmmu_get_page_size();
  1084. }
  1085. break;
  1086. }
  1087. }
  1088. mutex_unlock(&mem_man->mutex);
  1089. return ret;
  1090. }
  1091. EXPORT_SYMBOL(img_mmu_move_pg_to_cache);
  1092. static void _img_mem_sync_cpu_to_device(struct buffer *buffer, bool force)
  1093. {
  1094. struct mem_man *mem_man = &mem_man_data;
  1095. struct heap *heap = buffer->heap;
  1096. if (!cache_sync) {
  1097. pr_debug("%s:%d buffer %d size %zu cache synchronization disabled!\n",
  1098. __func__, __LINE__, buffer->id, buffer->actual_size);
  1099. return;
  1100. }
  1101. pr_debug("%s:%d buffer %d size %zu kptr %p cache(%d:%d)\n",
  1102. __func__, __LINE__, buffer->id, buffer->actual_size,
  1103. buffer->kptr, force, heap->cache_sync);
  1104. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  1105. if (heap->ops && heap->ops->sync_cpu_to_dev &&
  1106. (force || heap->cache_sync))
  1107. heap->ops->sync_cpu_to_dev(heap, buffer);
  1108. #ifdef CONFIG_ARM
  1109. dmb();
  1110. #else
  1111. /* Put memory barrier */
  1112. mb();
  1113. #endif
  1114. }
  1115. int img_mem_sync_cpu_to_device(struct mem_ctx *ctx, int buf_id)
  1116. {
  1117. struct mem_man *mem_man = &mem_man_data;
  1118. struct buffer *buffer;
  1119. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  1120. mutex_lock(&mem_man->mutex);
  1121. buffer = idr_find(&ctx->buffers, buf_id);
  1122. if (!buffer) {
  1123. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  1124. mutex_unlock(&mem_man->mutex);
  1125. return -EINVAL;
  1126. }
  1127. _img_mem_sync_cpu_to_device(buffer, false);
  1128. mutex_unlock(&mem_man->mutex);
  1129. return 0;
  1130. }
  1131. EXPORT_SYMBOL(img_mem_sync_cpu_to_device);
  1132. static void _img_mem_sync_device_to_cpu(struct buffer *buffer, bool force)
  1133. {
  1134. struct mem_man *mem_man = &mem_man_data;
  1135. struct heap *heap = buffer->heap;
  1136. if (!cache_sync) {
  1137. pr_debug("%s:%d buffer %d size %zu cache synchronization disabled!\n",
  1138. __func__, __LINE__, buffer->id, buffer->actual_size);
  1139. return;
  1140. }
  1141. pr_debug("%s:%d buffer %d size %zu kptr %p cache(%d:%d)\n",
  1142. __func__, __LINE__, buffer->id, buffer->actual_size,
  1143. buffer->kptr, force, heap->cache_sync);
  1144. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  1145. if (heap->ops && heap->ops->sync_dev_to_cpu &&
  1146. (force || heap->cache_sync))
  1147. heap->ops->sync_dev_to_cpu(heap, buffer);
  1148. }
  1149. int img_mem_sync_device_to_cpu(struct mem_ctx *ctx, int buf_id)
  1150. {
  1151. struct mem_man *mem_man = &mem_man_data;
  1152. struct buffer *buffer;
  1153. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  1154. mutex_lock(&mem_man->mutex);
  1155. buffer = idr_find(&ctx->buffers, buf_id);
  1156. if (!buffer) {
  1157. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  1158. mutex_unlock(&mem_man->mutex);
  1159. return -EINVAL;
  1160. }
  1161. _img_mem_sync_device_to_cpu(buffer, false);
  1162. mutex_unlock(&mem_man->mutex);
  1163. return 0;
  1164. }
  1165. EXPORT_SYMBOL(img_mem_sync_device_to_cpu);
  1166. int img_mem_get_usage(const struct mem_ctx *ctx, size_t *max, size_t *curr)
  1167. {
  1168. struct mem_man *mem_man = &mem_man_data;
  1169. mutex_lock(&mem_man->mutex);
  1170. if (max)
  1171. *max = ctx->mem_usage_max;
  1172. if (curr)
  1173. *curr = ctx->mem_usage_curr;
  1174. mutex_unlock(&mem_man->mutex);
  1175. return 0;
  1176. }
  1177. EXPORT_SYMBOL(img_mem_get_usage);
  1178. int img_mmu_get_usage(const struct mem_ctx *ctx, size_t *max, size_t *curr)
  1179. {
  1180. struct mem_man *mem_man = &mem_man_data;
  1181. mutex_lock(&mem_man->mutex);
  1182. if (max)
  1183. *max = ctx->mmu_usage_max;
  1184. if (curr)
  1185. *curr = ctx->mmu_usage_curr;
  1186. mutex_unlock(&mem_man->mutex);
  1187. return 0;
  1188. }
  1189. EXPORT_SYMBOL(img_mmu_get_usage);
  1190. static int img_mmu_cache_get_offset(struct mem_ctx *mem_ctx,
  1191. unsigned long addr, unsigned int *offset)
  1192. {
  1193. struct mmu_ctx *mmu_ctx;
  1194. list_for_each_entry(mmu_ctx, &mem_ctx->mmu_ctxs, mem_ctx_entry) {
  1195. if (addr >= mmu_ctx->cache_phys_start &&
  1196. addr < mmu_ctx->cache_phys_start + mmu_ctx->cache_size) {
  1197. *offset = addr - mmu_ctx->cache_phys_start;
  1198. return 0;
  1199. }
  1200. }
  1201. return -EINVAL;
  1202. }
  1203. /*
  1204. * related to stream MMU context (contains IMGMMU functionality in general)
  1205. */
  1206. static int imgmmu_find_buffer(struct mem_ctx *ctx, uint64_t addr,
  1207. int *buffer_id, unsigned int *buffer_offset)
  1208. {
  1209. struct heap *heap;
  1210. struct buffer *buffer;
  1211. int buf_id;
  1212. unsigned int buf_offset;
  1213. int ret;
  1214. for (buf_id = *buffer_id;
  1215. ((buffer) = idr_get_next(&ctx->buffers, &buf_id)) != NULL; ++buf_id) {
  1216. heap = buffer->heap;
  1217. if (heap->ops && heap->ops->get_sg_table) {
  1218. struct sg_table *sgt;
  1219. struct scatterlist *sgl;
  1220. bool use_sg_dma = false;
  1221. ret = heap->ops->get_sg_table(heap, buffer, &sgt, &use_sg_dma);
  1222. if (ret) {
  1223. pr_err("%s: heap %d buffer %d no sg_table!\n",
  1224. __func__, heap->id, buffer->id);
  1225. return -EINVAL;
  1226. }
  1227. if (buffer->pcache.last_sgl) {
  1228. sgl = buffer->pcache.last_sgl;
  1229. buf_offset = buffer->pcache.last_offset;
  1230. } else {
  1231. sgl = sgt->sgl;
  1232. buf_offset = 0;
  1233. }
  1234. while (sgl) {
  1235. phys_addr_t phys = use_sg_dma ?
  1236. sg_dma_address(sgl) : sg_phys(sgl);
  1237. unsigned int len = use_sg_dma ?
  1238. sg_dma_len(sgl) : sgl->length;
  1239. #if 0
  1240. pr_err("%s: phys %llx len:%d addr:%llx\n",
  1241. __func__, phys, len, addr);
  1242. #endif
  1243. if (phys == addr) {
  1244. #if 0
  1245. pr_err("%s: match @addr:%llx buf:%d offs:%d len:%d\n",
  1246. __func__, addr, buffer->id, buf_offset, len);
  1247. #endif
  1248. *buffer_id = buffer->id;
  1249. *buffer_offset = buf_offset;
  1250. return 0;
  1251. }
  1252. buffer->pcache.last_offset = buf_offset += len;
  1253. buffer->pcache.last_sgl = sgl = sg_next(sgl);
  1254. }
  1255. } else if (heap->ops && heap->ops->get_page_array) {
  1256. uint64_t *addrs;
  1257. int page_idx;
  1258. ret = heap->ops->get_page_array(heap, buffer, &addrs);
  1259. if (ret) {
  1260. pr_err("%s: heap %d buffer %d no page_array!\n",
  1261. __func__, heap->id, buffer->id);
  1262. return -EINVAL;
  1263. }
  1264. if (buffer->pcache.last_sgl) {
  1265. page_idx = buffer->pcache.last_idx;
  1266. buf_offset = buffer->pcache.last_offset;
  1267. } else {
  1268. page_idx = 0;
  1269. buf_offset = 0;
  1270. }
  1271. while (buf_offset < buffer->actual_size) {
  1272. if (addrs[page_idx] == addr) {
  1273. *buffer_id = buffer->id;
  1274. *buffer_offset = buf_offset;
  1275. return 0;
  1276. }
  1277. buffer->pcache.last_idx = page_idx++;
  1278. buffer->pcache.last_offset = buf_offset += PAGE_SIZE;
  1279. }
  1280. } else {
  1281. pr_err("%s: heap %d buffer %d no phys addrs found!\n",
  1282. __func__, heap->id, buffer->id);
  1283. return -EINVAL;
  1284. }
  1285. }
  1286. return -EINVAL;
  1287. }
  1288. static struct imgmmu_page *_page_alloc(void *arg, unsigned char type)
  1289. {
  1290. struct mem_man *mem_man = &mem_man_data;
  1291. struct mmu_ctx *mmu_ctx = arg;
  1292. struct mmu_page *page;
  1293. struct buffer *buffer;
  1294. struct heap *heap;
  1295. int ret;
  1296. pr_debug("%s:%d arg %p\n", __func__, __LINE__, arg);
  1297. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  1298. page = kzalloc(sizeof(struct mmu_page), GFP_KERNEL);
  1299. if (!page)
  1300. return NULL;
  1301. __img_pdump_printf(mmu_ctx->device, "-- Allocating MMU page for %s\n",
  1302. type == IMGMMU_PTYPE_PC ? "PC" :
  1303. type == IMGMMU_PTYPE_PD ? "PD" :
  1304. type == IMGMMU_PTYPE_PT ? "PT" :
  1305. "???");
  1306. ret = _img_mem_alloc(mmu_ctx->device, mmu_ctx->mem_ctx, mmu_ctx->heap,
  1307. imgmmu_get_cpu_page_size(), mmu_ctx->config.alloc_attr, &buffer);
  1308. if (ret) {
  1309. pr_err("%s: img_mem_alloc failed (%d)\n", __func__, ret);
  1310. goto free_page;
  1311. }
  1312. ret = _img_mem_map_km(buffer);
  1313. if (ret) {
  1314. pr_err("%s: img_mem_map_km failed (%d)\n", __func__, ret);
  1315. goto free_buffer;
  1316. }
  1317. WARN_ON(!type);
  1318. page->type = type;
  1319. page->buffer = buffer;
  1320. page->page.cpu_addr = (uintptr_t)buffer->kptr;
  1321. memset((void *)page->page.cpu_addr, 0, imgmmu_get_cpu_page_size());
  1322. if (type == IMGMMU_PTYPE_PT)
  1323. page->use_parity = mmu_ctx->config.use_pte_parity;
  1324. heap = buffer->heap;
  1325. if (heap->ops && heap->ops->get_sg_table) {
  1326. struct sg_table *sgt;
  1327. bool use_sg_dma = false;
  1328. ret = heap->ops->get_sg_table(heap, buffer, &sgt, &use_sg_dma);
  1329. if (ret) {
  1330. pr_err("%s: heap %d buffer %d no sg_table!\n",
  1331. __func__, heap->id, buffer->id);
  1332. ret = -EINVAL;
  1333. goto free_buffer;
  1334. }
  1335. if (use_sg_dma)
  1336. page->page.phys_addr = sg_dma_address(sgt->sgl);
  1337. else
  1338. page->page.phys_addr = sg_phys(sgt->sgl);
  1339. } else if (heap->ops && heap->ops->get_page_array) {
  1340. uint64_t *addrs;
  1341. ret = heap->ops->get_page_array(heap, buffer, &addrs);
  1342. if (ret) {
  1343. pr_err("%s: heap %d buffer %d no page array!\n",
  1344. __func__, heap->id, buffer->id);
  1345. ret = -EINVAL;
  1346. goto free_buffer;
  1347. }
  1348. page->page.phys_addr = *addrs; /* we allocated a single page */
  1349. } else {
  1350. pr_err("%s: heap %d buffer %d no get_sg or get_page_array!\n",
  1351. __func__, heap->id, buffer->id);
  1352. ret = -EINVAL;
  1353. goto free_buffer;
  1354. }
  1355. mmu_ctx->mem_ctx->mmu_usage_curr += buffer->actual_size;
  1356. if (mmu_ctx->mem_ctx->mmu_usage_curr > mmu_ctx->mem_ctx->mmu_usage_max)
  1357. mmu_ctx->mem_ctx->mmu_usage_max = mmu_ctx->mem_ctx->mmu_usage_curr;
  1358. pr_debug("%s:%d virt addr %#lx type:%d\n", __func__, __LINE__,
  1359. page->page.cpu_addr, type);
  1360. pr_debug("%s:%d phys addr %#llx\n", __func__, __LINE__,
  1361. page->page.phys_addr);
  1362. return &page->page;
  1363. free_buffer:
  1364. _img_mem_free(buffer);
  1365. free_page:
  1366. kfree(page);
  1367. return NULL;
  1368. }
  1369. static void _page_free(struct imgmmu_page *arg)
  1370. {
  1371. struct mem_man *mem_man = &mem_man_data;
  1372. struct mmu_page *page;
  1373. page = container_of(arg, struct mmu_page, page);
  1374. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  1375. pr_debug("%s:%d buffer %u\n", __func__, __LINE__, page->buffer->id);
  1376. if (page->buffer->mem_ctx->mmu_usage_curr >= page->buffer->actual_size)
  1377. page->buffer->mem_ctx->mmu_usage_curr -= page->buffer->actual_size;
  1378. else
  1379. WARN_ON(1);
  1380. _img_mem_free(page->buffer);
  1381. kfree(page);
  1382. }
  1383. static inline void __pdump_apply_parity(struct device* dev, uint64_t virt,
  1384. const char *block, unsigned int offset)
  1385. {
  1386. uint8_t bits;
  1387. /* XOR 32 bit pair <paddr & vaadr> */
  1388. __img_pdump_printf(dev,
  1389. "WRW "_PMEM_":$1 %#llx -- Calculate parity bit\n"
  1390. "WRW "_PMEM_":$2 %s:%#x\n"
  1391. "SHR "_PMEM_":$2 "_PMEM_":$2 %d\n"
  1392. "XOR "_PMEM_":$1 "_PMEM_":$1 "_PMEM_":$2\n",
  1393. virt >> MIN_PAGE_SIZE_BITS, block, offset, MIN_PAGE_SIZE_BITS);
  1394. for (bits = 16; bits >= 1; bits>>=1)
  1395. /* XOR 'bits' pair of previous result */
  1396. __img_pdump_printf(dev,
  1397. "AND "_PMEM_":$2 "_PMEM_":$1 %#x\n"
  1398. "SHR "_PMEM_":$1 "_PMEM_":$1 %d\n"
  1399. "XOR "_PMEM_":$1 "_PMEM_":$1 "_PMEM_":$2\n",
  1400. (1<<bits)-1, bits);
  1401. /* Apply parity bit */
  1402. __img_pdump_printf(dev,
  1403. "SHL "_PMEM_":$1 "_PMEM_":$1 %d\n"
  1404. "OR "_PMEM_":$0 "_PMEM_":$0 "_PMEM_":$1 -- Apply parity\n",
  1405. imgmmu_get_pte_parity_shift());
  1406. }
  1407. static void _page_write(struct imgmmu_page *page,
  1408. unsigned int offset, uint64_t entry,
  1409. unsigned int flags, void *priv)
  1410. {
  1411. uint64_t *mem64 = (uint64_t *)page->cpu_addr;
  1412. uint32_t *mem32 = (uint32_t *)mem64;
  1413. struct mmu_page *mmu_page;
  1414. struct heap *heap = NULL;
  1415. struct buffer *buf = (struct buffer*)priv;
  1416. struct pdump_descr* pdump;
  1417. uint32_t entry_shift = 0;
  1418. uint64_t cache_bits = 0;
  1419. uint64_t address = entry & IMG_MMU_PHY_ADDR_MASK;
  1420. uint64_t virt = page->virt_base;
  1421. uint64_t paddr = entry;
  1422. mmu_page = container_of(page, struct mmu_page, page);
  1423. virt += ((1<<imgmmu_get_entry_shift(mmu_page->type))) * offset;
  1424. if (mmu_page->type == IMGMMU_PTYPE_PC ||
  1425. mmu_page->type == IMGMMU_PTYPE_PD)
  1426. heap = mmu_page->buffer->heap;
  1427. else {
  1428. /* PT entries are pointing to buffer which may have been allocated
  1429. * using different heap than the one used for mmu allocations */
  1430. if (buf)
  1431. heap = buf->heap;
  1432. }
  1433. mmu_page->bypass_addr_trans = (flags & IMGMMU_BYPASS_ADDR_TRANS ? true : false);
  1434. /* Mask MMU flags */
  1435. flags &= IMG_MMU_ENTRY_FLAGS_MASK;
  1436. /* skip translation when flags are zero, assuming address is invalid */
  1437. /* or when page is being remapped to on-chip ram*/
  1438. if (flags && heap && heap->to_dev_addr &&
  1439. !mmu_page->bypass_addr_trans)
  1440. paddr = heap->to_dev_addr(&heap->options, paddr);
  1441. WARN(paddr & ~dma_get_mask(mmu_page->buffer->device),
  1442. "%s: Physical address is out of dma mask, "\
  1443. "Set proper dma mask to avoid cache problems, ", __func__);
  1444. WARN(paddr & ~IMG_MMU_PHY_ADDR_MASK,
  1445. "%s: Physical address exceeds hardware capabilities, "\
  1446. "MMU mapping will be likely invalid", __func__);
  1447. paddr &= IMG_MMU_PHY_ADDR_MASK;
  1448. if (trace_physical_pages && flags) {
  1449. if (mmu_page->type == IMGMMU_PTYPE_PC)
  1450. pr_info("%s: sid:%d off %#03x paddr %#016llx flags %#x type:PC\n",
  1451. __func__, buf ? buf->mem_ctx->id : -1, offset, paddr, flags);
  1452. else
  1453. pr_info("%s: sid:%d off %#03x paddr %#016llx flags %#x type:%s virt:%llx\n",
  1454. __func__, buf ? buf->mem_ctx->id : -1, offset, paddr, flags,
  1455. mmu_page->type == IMGMMU_PTYPE_PD ? "PD" :
  1456. mmu_page->type == IMGMMU_PTYPE_PT ? "PT" :
  1457. "???", virt);
  1458. }
  1459. if (mmu_page->type == IMGMMU_PTYPE_PC) {
  1460. /* Offset of PD physical base address(12)-4(PCE flags) */
  1461. entry_shift = 8;
  1462. /* This is 32 bit entry */
  1463. mem32[offset] = (paddr >> entry_shift) | flags;
  1464. } else if (mmu_page->type == IMGMMU_PTYPE_PD) {
  1465. /* Offset of PT physical base address(12)-12(PDE flags) */
  1466. entry_shift = 0;
  1467. /* This is 64 bit entry */
  1468. mem64[offset] = paddr | flags;
  1469. } else if (mmu_page->type == IMGMMU_PTYPE_PT) {
  1470. /* Offset of page physical base address(12)-12(PTE flags) */
  1471. entry_shift = 0;
  1472. cache_bits = imgmmu_get_pte_cache_bits(entry);
  1473. /* This is 64 bit entry */
  1474. mem64[offset] = cache_bits | paddr | flags;
  1475. if (flags && mmu_page->use_parity) {
  1476. uint64_t par_pair = (virt >> MIN_PAGE_SIZE_BITS) |
  1477. ((paddr >> MIN_PAGE_SIZE_BITS) << (sizeof(uint32_t)*8));
  1478. bool par_bit = img_mem_calc_parity(par_pair);
  1479. if (par_bit)
  1480. imgmmu_set_pte_parity(&mem64[offset]);
  1481. if (trace_physical_pages)
  1482. pr_info("%s: [%llx]: %s\n", __func__, mem64[offset],
  1483. par_bit ? "odd parity" : "even parity");
  1484. }
  1485. }
  1486. pdump = vha_pdump_dev_get_drvdata(mmu_page->buffer->device);
  1487. if (img_pdump_enabled(pdump) && flags) {
  1488. /* skip when flags are zero, assuming address is invalid */
  1489. int buffer_id = 0;
  1490. unsigned int buffer_offset = 0;
  1491. int ret;
  1492. if (mmu_page->bypass_addr_trans) {
  1493. ret = img_mmu_cache_get_offset(mmu_page->buffer->mem_ctx, address,
  1494. &buffer_offset);
  1495. if (ret) {
  1496. pr_info("PDUMP: Can't find in cache %#llx\n", address);
  1497. } else {
  1498. /* Cache addresses are only applicable for PT entries */
  1499. WARN_ON(mmu_page->type != IMGMMU_PTYPE_PT);
  1500. __img_pdump_printf(mmu_page->buffer->device,
  1501. "WRW "_PMEM_":$0 :OCM:BLOCK_CACHE:%#x\n"
  1502. "OR "_PMEM_":$0 "_PMEM_":$0 %d\n",
  1503. buffer_offset, flags);
  1504. if (mmu_page->use_parity) {
  1505. const char block[] = ":OCM:BLOCK_CACHE";
  1506. __pdump_apply_parity(mmu_page->buffer->device, virt, block, buffer_offset);
  1507. }
  1508. if (cache_bits)
  1509. __img_pdump_printf(mmu_page->buffer->device,
  1510. "OR "_PMEM_":$0 "_PMEM_":$0 %#llx\n",
  1511. cache_bits);
  1512. __img_pdump_printf(mmu_page->buffer->device,
  1513. "WRW64 "_PMEM_":BLOCK_%d:%#zx "_PMEM_":$0 -- PTE\n",
  1514. mmu_page->buffer->id, offset * sizeof(*mem64));
  1515. }
  1516. } else {
  1517. if (mmu_page->type == IMGMMU_PTYPE_PT && buf)
  1518. buffer_id = buf->id;
  1519. ret = imgmmu_find_buffer(mmu_page->buffer->mem_ctx, address,
  1520. &buffer_id, &buffer_offset);
  1521. if (ret) {
  1522. pr_info("PDUMP: Can't find %#llx\n", address);
  1523. } else if (mmu_page->type == IMGMMU_PTYPE_PC) {
  1524. __img_pdump_printf(mmu_page->buffer->device,
  1525. "WRW "_PMEM_":$0 "_PMEM_":BLOCK_%d:%#x\n"
  1526. "SHR "_PMEM_":$0 "_PMEM_":$0 %d\n"
  1527. "OR "_PMEM_":$0 "_PMEM_":$0 %d\n"
  1528. "WRW "_PMEM_":BLOCK_%d:%#zx "_PMEM_":$0 -- PCE\n",
  1529. buffer_id, buffer_offset,
  1530. entry_shift, flags,
  1531. mmu_page->buffer->id, offset * sizeof(*mem32));
  1532. } else {
  1533. if (mmu_page->type == IMGMMU_PTYPE_PD) {
  1534. __img_pdump_printf(mmu_page->buffer->device,
  1535. "WRW "_PMEM_":$0 "_PMEM_":BLOCK_%d:%#x\n"
  1536. "OR "_PMEM_":$0 "_PMEM_":$0 %d\n"
  1537. "WRW64 "_PMEM_":BLOCK_%d:%#zx "_PMEM_":$0 -- PDE\n",
  1538. buffer_id, buffer_offset, flags,
  1539. mmu_page->buffer->id, offset * sizeof(*mem64));
  1540. } else if (mmu_page->type == IMGMMU_PTYPE_PT) {
  1541. char block[25];
  1542. if (heap->type == IMG_MEM_HEAP_TYPE_OCM)
  1543. snprintf(block, sizeof(block), ":OCM:BLOCK_CACHE");
  1544. else
  1545. snprintf(block, sizeof(block), ""_PMEM_":BLOCK_%d",
  1546. buffer_id);
  1547. __img_pdump_printf(mmu_page->buffer->device,
  1548. "WRW "_PMEM_":$0 %s:%#x\n"
  1549. "OR "_PMEM_":$0 "_PMEM_":$0 %d\n",
  1550. block, buffer_offset, flags);
  1551. if (mmu_page->use_parity)
  1552. __pdump_apply_parity(mmu_page->buffer->device, virt, block, buffer_offset);
  1553. if (cache_bits)
  1554. __img_pdump_printf(mmu_page->buffer->device,
  1555. "OR "_PMEM_":$0 "_PMEM_":$0 %#llx\n",
  1556. cache_bits);
  1557. __img_pdump_printf(mmu_page->buffer->device,
  1558. "WRW64 "_PMEM_":BLOCK_%d:%#zx "_PMEM_":$0 -- PTE\n",
  1559. mmu_page->buffer->id, offset * sizeof(*mem64));
  1560. }
  1561. }
  1562. }
  1563. }
  1564. }
  1565. static uint64_t _page_read(struct imgmmu_page *page,
  1566. unsigned int offset, void *priv,
  1567. unsigned int *flags)
  1568. {
  1569. uint64_t *mem64 = (uint64_t *)page->cpu_addr;
  1570. uint32_t *mem32 = (uint32_t *)mem64;
  1571. struct mmu_page *mmu_page;
  1572. struct heap *heap = NULL;
  1573. uint32_t entry_shift = 0;
  1574. uint64_t entry = 0;
  1575. uint64_t paddr;
  1576. uint64_t virt = page->virt_base;
  1577. struct buffer *buf = (struct buffer*)priv;
  1578. mmu_page = container_of(page, struct mmu_page, page);
  1579. virt += ((1<<imgmmu_get_entry_shift(mmu_page->type))) * offset;
  1580. if (mmu_page->type == IMGMMU_PTYPE_PC ||
  1581. mmu_page->type == IMGMMU_PTYPE_PD)
  1582. heap = mmu_page->buffer->heap;
  1583. else {
  1584. /* PT entries are pointing to buffer which may have been allocated
  1585. * using different heap than the one used for mmu allocations */
  1586. if (buf)
  1587. heap = buf->heap;
  1588. }
  1589. if (mmu_page->type == IMGMMU_PTYPE_PC) {
  1590. /* Offset of PD physical base address(12)-4(PCE flags) */
  1591. entry_shift = 8;
  1592. /* This is 32 bit entry */
  1593. entry = mem32[offset];
  1594. } else if (mmu_page->type == IMGMMU_PTYPE_PD) {
  1595. /* Offset of PT physical base address(12)-12(PDE flags) */
  1596. entry_shift = 0;
  1597. /* This is 64 bit entry */
  1598. entry = mem64[offset];
  1599. } else if (mmu_page->type == IMGMMU_PTYPE_PT) {
  1600. /* Offset of page physical base address(12)-12(PTE flags) */
  1601. entry_shift = 0;
  1602. /* This is 64 bit entry */
  1603. entry = mem64[offset];
  1604. }
  1605. *flags = entry & IMG_MMU_ENTRY_FLAGS_MASK;
  1606. paddr = entry & ~IMG_MMU_ENTRY_FLAGS_MASK;
  1607. if (mmu_page->type == IMGMMU_PTYPE_PT) {
  1608. /* Mask parity and special cache bits */
  1609. paddr &= ~(1ULL<<imgmmu_get_pte_parity_shift());
  1610. paddr &= ~imgmmu_get_pte_cache_bits(entry);
  1611. }
  1612. paddr <<= entry_shift;
  1613. /* Check if physical address set in PTE is within correct range */
  1614. if (paddr & ~IMG_MMU_PHY_ADDR_MASK) {
  1615. pr_err("%s: mmu page entry (%llx) corruption detected (phys)!\n",
  1616. __func__, paddr);
  1617. *flags = IMG_MMU_ENTRY_FLAGS_MASK;
  1618. goto exit;
  1619. }
  1620. /* Check parity */
  1621. if (*flags && mmu_page->type == IMGMMU_PTYPE_PT &&
  1622. mmu_page->use_parity) {
  1623. uint64_t par_pair = (virt >> MIN_PAGE_SIZE_BITS) |
  1624. ((paddr >> MIN_PAGE_SIZE_BITS) << (sizeof(uint32_t)*8));
  1625. bool par_bit = img_mem_calc_parity(par_pair);
  1626. if (trace_physical_pages)
  1627. pr_info("%s: [%llx]: %s\n", __func__, entry,
  1628. par_bit ? "odd parity" : "even parity");
  1629. if ((entry >> imgmmu_get_pte_parity_shift()) != par_bit) {
  1630. pr_err("%s: mmu page entry (%llx) corruption detected (parity)!\n",
  1631. __func__, entry);
  1632. *flags = IMG_MMU_ENTRY_FLAGS_MASK;
  1633. goto exit;
  1634. }
  1635. }
  1636. /* skip translation when flags are zero, assuming address is invalid */
  1637. if (*flags && heap && heap->to_host_addr &&
  1638. !mmu_page->bypass_addr_trans)
  1639. paddr = heap->to_host_addr(&heap->options, paddr);
  1640. /* Check if physical address matches dma mask */
  1641. if (paddr & ~dma_get_mask(mmu_page->buffer->device)) {
  1642. pr_err("%s: mmu page entry (%llx) physical address is out of dma mask!\n"
  1643. "Set proper dma mask to avoid cache problems\n",
  1644. __func__, paddr);
  1645. *flags = IMG_MMU_ENTRY_FLAGS_MASK;
  1646. goto exit;
  1647. }
  1648. /* Sanity check for MMU flags - different on each level */
  1649. if (((mmu_page->type == IMGMMU_PTYPE_PC ||
  1650. mmu_page->type == IMGMMU_PTYPE_PD) &&
  1651. (*flags & ~IMG_MMU_PTE_FLAG_VALID)) ||
  1652. (mmu_page->type == IMGMMU_PTYPE_PT &&
  1653. (*flags & ~(IMG_MMU_PTE_FLAG_VALID|IMG_MMU_PTE_FLAG_READ_ONLY)))) {
  1654. pr_err("%s: mmu page entry corruption detected (flags)!\n",
  1655. __func__);
  1656. *flags = IMG_MMU_ENTRY_FLAGS_MASK;
  1657. }
  1658. exit:
  1659. if (trace_physical_pages && *flags) {
  1660. if (mmu_page->type == IMGMMU_PTYPE_PC)
  1661. pr_info("%s: sid:%d off %#03x paddr %#016llx flags %#x type:PC\n",
  1662. __func__, buf ? buf->mem_ctx->id : -1, offset, paddr, *flags);
  1663. else
  1664. pr_info("%s: sid:%d off %#03x paddr %#016llx flags %#x type:%s virt:%llx\n",
  1665. __func__, buf ? buf->mem_ctx->id : -1, offset, paddr, *flags,
  1666. mmu_page->type == IMGMMU_PTYPE_PD ? "PD" :
  1667. mmu_page->type == IMGMMU_PTYPE_PT ? "PT" :
  1668. "???", virt);
  1669. }
  1670. return paddr;
  1671. }
  1672. static void _update_page(struct imgmmu_page *arg)
  1673. {
  1674. struct mem_man *mem_man = &mem_man_data;
  1675. struct mmu_page *page;
  1676. if (trace_physical_pages)
  1677. pr_debug("%s\n", __func__);
  1678. page = container_of(arg, struct mmu_page, page);
  1679. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  1680. _img_mem_sync_cpu_to_device(page->buffer, true);
  1681. }
  1682. int img_mmu_ctx_create(struct device *device, const struct mmu_config *config,
  1683. struct mem_ctx *mem_ctx, int heap_id,
  1684. int (*callback_fn)(enum img_mmu_callback_type type,
  1685. int buf_id, void *data),
  1686. void *callback_data, struct mmu_ctx **mmu_ctx)
  1687. {
  1688. struct mem_man *mem_man = &mem_man_data;
  1689. static struct imgmmu_info info = {
  1690. .page_alloc = _page_alloc,
  1691. .page_free = _page_free,
  1692. .page_write = _page_write,
  1693. .page_read = _page_read,
  1694. .page_update = _update_page,
  1695. };
  1696. struct mmu_ctx *ctx;
  1697. int res;
  1698. if (((config->addr_width < imgmmu_get_virt_size()) ||
  1699. (config->addr_width < imgmmu_get_phys_size())) &&
  1700. !config->bypass_hw) {
  1701. pr_err("%s: invalid addr_width (%d)!\n",
  1702. __func__, config->addr_width);
  1703. return -EINVAL;
  1704. }
  1705. ctx = kzalloc(sizeof(struct mmu_ctx), GFP_KERNEL);
  1706. if (!ctx)
  1707. return -ENOMEM;
  1708. ctx->device = device;
  1709. ctx->mem_ctx = mem_ctx;
  1710. memcpy(&ctx->config, config, sizeof(struct mmu_config));
  1711. imgmmu_set_page_size(config->page_size);
  1712. mutex_lock(&mem_man->mutex);
  1713. ctx->heap = idr_find(&mem_man->heaps, heap_id);
  1714. if (!ctx->heap) {
  1715. pr_err("%s: invalid heap_id (%d)!\n", __func__, heap_id);
  1716. mutex_unlock(&mem_man->mutex);
  1717. kfree(ctx);
  1718. return -EINVAL;
  1719. }
  1720. /* Apply offset when needed */
  1721. if (ctx->heap->ops->set_offset) {
  1722. if (ctx->heap->ops->set_offset(ctx->heap, config->bypass_offset)) {
  1723. pr_err("%s: failed to set offset %zu heap_id (%d)!\n",
  1724. __func__, config->bypass_offset, heap_id);
  1725. mutex_unlock(&mem_man->mutex);
  1726. kfree(ctx);
  1727. return -EINVAL;
  1728. }
  1729. pr_debug("%s adding %lx offset bytes to heap %d type %d (%s)\n",
  1730. __func__, config->bypass_offset, ctx->heap->id,
  1731. ctx->heap->type, get_heap_name(ctx->heap->type));
  1732. }
  1733. info.ctx = ctx;
  1734. /* If we are in bypass mode, do not populate hw structures */
  1735. if (!config->bypass_hw) {
  1736. ctx->mmu_cat = imgmmu_cat_create(&info, &res);
  1737. if (res) {
  1738. pr_err("%s: catalogue create failed (%d)!\n",
  1739. __func__, res);
  1740. mutex_unlock(&mem_man->mutex);
  1741. kfree(ctx);
  1742. return -EFAULT;
  1743. }
  1744. } else
  1745. pr_debug("%s imgmmu_cat_create bypass!\n", __func__);
  1746. list_add(&ctx->mem_ctx_entry, &mem_ctx->mmu_ctxs);
  1747. INIT_LIST_HEAD(&ctx->mappings);
  1748. ctx->callback_fn = callback_fn;
  1749. ctx->callback_data = callback_data;
  1750. ctx->id = mem_ctx->id;
  1751. *mmu_ctx = ctx;
  1752. mutex_unlock(&mem_man->mutex);
  1753. return ctx->id;
  1754. }
  1755. EXPORT_SYMBOL(img_mmu_ctx_create);
  1756. static void _img_mmu_ctx_destroy(struct mmu_ctx *ctx)
  1757. {
  1758. struct mem_man *mem_man = &mem_man_data;
  1759. int res;
  1760. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  1761. while (!list_empty(&ctx->mappings)) {
  1762. struct mmu_ctx_mapping *map;
  1763. map = list_first_entry(&ctx->mappings,
  1764. struct mmu_ctx_mapping, mmu_ctx_entry);
  1765. pr_info("%s: found mapped buffer %d (size %zu)\n",
  1766. __func__, map->buffer->id, map->buffer->request_size);
  1767. _img_mmu_unmap(map);
  1768. kfree(map);
  1769. }
  1770. if (!ctx->config.bypass_hw) {
  1771. res = imgmmu_cat_destroy(ctx->mmu_cat);
  1772. if (res)
  1773. pr_err("imgmmu_cat_destroy failed (%d)!\n", res);
  1774. } else
  1775. pr_debug("%s imgmmu_cat_destroy bypass!\n", __func__);
  1776. list_del(&ctx->mem_ctx_entry);
  1777. }
  1778. void img_mmu_ctx_destroy(struct mmu_ctx *ctx)
  1779. {
  1780. struct mem_man *mem_man = &mem_man_data;
  1781. mutex_lock(&mem_man->mutex);
  1782. _img_mmu_ctx_destroy(ctx);
  1783. mutex_unlock(&mem_man->mutex);
  1784. kfree(ctx);
  1785. }
  1786. EXPORT_SYMBOL(img_mmu_ctx_destroy);
  1787. int img_mmu_map(struct mmu_ctx *mmu_ctx, struct mem_ctx *mem_ctx, int buf_id,
  1788. uint64_t virt_addr, unsigned int map_flags)
  1789. {
  1790. struct mem_man *mem_man = &mem_man_data;
  1791. struct mmu_ctx_mapping *mapping;
  1792. struct imgmmu_halloc heap_alloc;
  1793. struct buffer *buffer;
  1794. struct heap *heap;
  1795. int res = 0;
  1796. int ret;
  1797. pr_debug("%s buffer %d virt_addr %#llx\n",
  1798. __func__, buf_id, virt_addr);
  1799. mapping = kzalloc(sizeof(struct mmu_ctx_mapping), GFP_KERNEL);
  1800. if (!mapping)
  1801. return -ENOMEM;
  1802. mutex_lock(&mem_man->mutex);
  1803. buffer = idr_find(&mem_ctx->buffers, buf_id);
  1804. if (!buffer) {
  1805. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  1806. ret = -EINVAL;
  1807. goto error;
  1808. }
  1809. pr_debug("%s buffer %d 0x%p size %zu virt_addr %#llx\n", __func__,
  1810. buf_id, buffer, buffer->request_size, virt_addr);
  1811. /* Store MMU mapping flags */
  1812. buffer->map_flags = map_flags;
  1813. heap_alloc.vaddr = virt_addr;
  1814. heap_alloc.size = buffer->actual_size;
  1815. mapping->mmu_ctx = mmu_ctx;
  1816. mapping->buffer = buffer;
  1817. mapping->virt_addr = virt_addr;
  1818. if (!mmu_ctx->config.bypass_hw)
  1819. __img_pdump_printf(buffer->device, "-- Mapping "_PMEM_":BLOCK_%d @ 0x%llx\n",
  1820. buf_id, virt_addr);
  1821. heap = buffer->heap;
  1822. if (heap->ops && heap->ops->get_sg_table) {
  1823. struct sg_table *sgt;
  1824. bool use_sg_dma = false;
  1825. ret = heap->ops->get_sg_table(heap, buffer, &sgt, &use_sg_dma);
  1826. if (ret) {
  1827. pr_err("%s: heap %d buffer %d no sg_table!\n",
  1828. __func__, heap->id, buffer->id);
  1829. goto error;
  1830. }
  1831. if (!mmu_ctx->config.bypass_hw)
  1832. mapping->map = imgmmu_cat_map_sg(
  1833. mmu_ctx->mmu_cat,
  1834. sgt->sgl,
  1835. use_sg_dma,
  1836. &heap_alloc,
  1837. map_flags, buffer,
  1838. &res);
  1839. else
  1840. pr_debug("%s imgmmu_cat_map_sg bypass!\n", __func__);
  1841. } else if (heap->ops && heap->ops->get_page_array) {
  1842. uint64_t *addrs;
  1843. ret = heap->ops->get_page_array(heap, buffer, &addrs);
  1844. if (ret) {
  1845. pr_err("%s: heap %d buffer %d no page array!\n",
  1846. __func__, heap->id, buffer->id);
  1847. goto error;
  1848. }
  1849. if (!mmu_ctx->config.bypass_hw)
  1850. mapping->map = imgmmu_cat_map_arr(
  1851. mmu_ctx->mmu_cat,
  1852. addrs,
  1853. &heap_alloc,
  1854. map_flags, buffer,
  1855. &res);
  1856. else
  1857. pr_debug("%s imgmmu_cat_map_arr bypass!\n", __func__);
  1858. } else {
  1859. pr_err("%s: heap %d buffer %d no get_sg or get_page_array!\n",
  1860. __func__, heap->id, buffer->id);
  1861. ret = -EINVAL;
  1862. goto error;
  1863. }
  1864. if (res) {
  1865. pr_err("imgmmu_cat_map failed (%d)!\n", res);
  1866. ret = -EFAULT;
  1867. goto error;
  1868. }
  1869. list_add(&mapping->mmu_ctx_entry, &mmu_ctx->mappings);
  1870. list_add(&mapping->buffer_entry, &mapping->buffer->mappings);
  1871. if (mmu_ctx->callback_fn && !mmu_ctx->config.bypass_hw) {
  1872. ret = mmu_ctx->callback_fn(IMG_MMU_CALLBACK_MAP, buffer->id,
  1873. mmu_ctx->callback_data);
  1874. if (ret) {
  1875. pr_err("%s: imgmmu map callback failed!\n", __func__);
  1876. }
  1877. }
  1878. mutex_unlock(&mem_man->mutex);
  1879. return ret;
  1880. error:
  1881. mutex_unlock(&mem_man->mutex);
  1882. kfree(mapping);
  1883. return ret;
  1884. }
  1885. EXPORT_SYMBOL(img_mmu_map);
  1886. static void _img_mmu_unmap(struct mmu_ctx_mapping *mapping)
  1887. {
  1888. struct mem_man *mem_man = &mem_man_data;
  1889. struct mmu_ctx *ctx = mapping->mmu_ctx;
  1890. int res;
  1891. pr_debug("%s:%d unmapping %p buffer %d\n",
  1892. __func__, __LINE__, mapping, mapping->buffer->id);
  1893. WARN_ON(!mutex_is_locked(&mem_man->mutex));
  1894. if (!ctx->config.bypass_hw) {
  1895. res = imgmmu_cat_unmap(mapping->map);
  1896. if (res)
  1897. pr_warn("imgmmu_cat_unmap failed (%d)!\n", res);
  1898. } else
  1899. pr_debug("%s imgmmu_cat_unmap bypass!\n", __func__);
  1900. list_del(&mapping->mmu_ctx_entry);
  1901. list_del(&mapping->buffer_entry);
  1902. if (ctx->callback_fn && !ctx->config.bypass_hw)
  1903. ctx->callback_fn(IMG_MMU_CALLBACK_UNMAP, mapping->buffer->id,
  1904. ctx->callback_data);
  1905. }
  1906. int img_mmu_unmap(struct mmu_ctx *mmu_ctx, struct mem_ctx *mem_ctx, int buf_id)
  1907. {
  1908. struct mem_man *mem_man = &mem_man_data;
  1909. struct mmu_ctx_mapping *mapping;
  1910. struct list_head *lst;
  1911. pr_debug("%s:%d buffer %d\n", __func__, __LINE__, buf_id);
  1912. mutex_lock(&mem_man->mutex);
  1913. mapping = NULL;
  1914. list_for_each(lst, &mmu_ctx->mappings) {
  1915. struct mmu_ctx_mapping *m;
  1916. m = list_entry(lst, struct mmu_ctx_mapping, mmu_ctx_entry);
  1917. if (m->buffer->id == buf_id) {
  1918. mapping = m;
  1919. break;
  1920. }
  1921. }
  1922. if (!mapping) {
  1923. pr_err("%s: buffer id %d not found\n", __func__, buf_id);
  1924. mutex_unlock(&mem_man->mutex);
  1925. return -EINVAL;
  1926. }
  1927. _img_mmu_unmap(mapping);
  1928. mutex_unlock(&mem_man->mutex);
  1929. kfree(mapping);
  1930. return 0;
  1931. }
  1932. EXPORT_SYMBOL(img_mmu_unmap);
  1933. int img_mmu_get_conf(size_t *page_size, size_t *virt_size)
  1934. {
  1935. if (page_size)
  1936. *page_size = imgmmu_get_page_size();
  1937. if (virt_size)
  1938. *virt_size = imgmmu_get_virt_size();
  1939. return 0;
  1940. }
  1941. EXPORT_SYMBOL(img_mmu_get_conf);
  1942. int img_mmu_get_pc(const struct mmu_ctx *ctx, unsigned int *pc_reg, int *bufid)
  1943. {
  1944. struct mem_man *mem_man = &mem_man_data;
  1945. struct imgmmu_page *page = NULL;
  1946. phys_addr_t addr = 0ULL;
  1947. mutex_lock(&mem_man->mutex);
  1948. *pc_reg = 0;
  1949. if (!ctx->config.bypass_hw) {
  1950. struct mmu_page *mmu_page;
  1951. page = imgmmu_cat_get_page(ctx->mmu_cat);
  1952. if (!page) {
  1953. mutex_unlock(&mem_man->mutex);
  1954. return -EINVAL;
  1955. }
  1956. mmu_page = container_of(page, struct mmu_page, page);
  1957. *bufid = mmu_page->buffer->id;
  1958. addr = page->phys_addr;
  1959. if (ctx->heap->to_dev_addr)
  1960. addr = ctx->heap->to_dev_addr(
  1961. &ctx->heap->options,
  1962. addr);
  1963. /* This is PFN of Page Catalogue phy address */
  1964. *pc_reg = (unsigned int)(addr >>= IMG_MMU_PC_ADDR_SHIFT);
  1965. pr_debug("%s: addr %#llx pc %#llx bufid %d\n", __func__,
  1966. page->phys_addr, addr, *bufid);
  1967. } else
  1968. pr_debug("%s imgmmu_cat_get_page bypass!\n", __func__);
  1969. mutex_unlock(&mem_man->mutex);
  1970. return 0;
  1971. }
  1972. EXPORT_SYMBOL(img_mmu_get_pc);
  1973. phys_addr_t img_mmu_get_paddr(const struct mmu_ctx *ctx,
  1974. uint64_t vaddr, uint8_t *flags)
  1975. {
  1976. struct mem_man *mem_man = &mem_man_data;
  1977. uint64_t entry = 0;
  1978. phys_addr_t paddr = 0;
  1979. *flags = 0;
  1980. mutex_lock(&mem_man->mutex);
  1981. entry = imgmmu_cat_get_pte(ctx->mmu_cat, vaddr);
  1982. if (entry != ~0) {
  1983. *flags = entry & IMG_MMU_ENTRY_FLAGS_MASK;
  1984. paddr = entry & ~IMG_MMU_ENTRY_FLAGS_MASK;
  1985. }
  1986. mutex_unlock(&mem_man->mutex);
  1987. return paddr;
  1988. }
  1989. EXPORT_SYMBOL(img_mmu_get_paddr);
  1990. /*
  1991. * Wrapper functions for virtual address allocator
  1992. */
  1993. int img_mmu_vaa_create(struct device *device,
  1994. uint32_t base, size_t size, struct mmu_vaa **vaa)
  1995. {
  1996. struct mem_man *mem_man = &mem_man_data;
  1997. struct mmu_vaa *ctx;
  1998. int ret = 0;
  1999. if (!size)
  2000. return -EINVAL;
  2001. ctx = kzalloc(sizeof(struct mmu_vaa), GFP_KERNEL);
  2002. if (!ctx)
  2003. return -ENOMEM;
  2004. mutex_lock(&mem_man->mutex);
  2005. ctx->heap = imgmmu_hcreate(base, imgmmu_get_page_size(),
  2006. size, true, &ret);
  2007. if (ret) {
  2008. pr_err("%s: imgmmu_hcreate failed (%d)!\n", __func__, ret);
  2009. kfree(ctx);
  2010. ret = -EFAULT;
  2011. goto exit;
  2012. }
  2013. INIT_LIST_HEAD(&ctx->entries);
  2014. ctx->device = device;
  2015. *vaa = ctx;
  2016. exit:
  2017. mutex_unlock(&mem_man->mutex);
  2018. return ret;
  2019. }
  2020. EXPORT_SYMBOL(img_mmu_vaa_create);
  2021. int img_mmu_vaa_destroy(struct mmu_vaa *vaa)
  2022. {
  2023. struct mem_man *mem_man = &mem_man_data;
  2024. if (!vaa)
  2025. return -EINVAL;
  2026. mutex_lock(&mem_man->mutex);
  2027. while (!list_empty(&vaa->entries)) {
  2028. struct vaa_entry *entry;
  2029. entry = list_first_entry(&vaa->entries,
  2030. struct vaa_entry, mmu_vaa_entry);
  2031. if (imgmmu_hfree(entry->alloc)) {
  2032. pr_err("%s: imgmmu_hfree failed!\n",
  2033. __func__);
  2034. WARN_ON(1);
  2035. }
  2036. list_del(&entry->mmu_vaa_entry);
  2037. kfree(entry);
  2038. }
  2039. if (imgmmu_hdestroy(vaa->heap)) {
  2040. pr_err("%s: imgmmu_hdestroy failed!\n", __func__);
  2041. /* If some attachments are still active */
  2042. WARN_ON(1);
  2043. }
  2044. kfree(vaa);
  2045. mutex_unlock(&mem_man->mutex);
  2046. return 0;
  2047. }
  2048. EXPORT_SYMBOL(img_mmu_vaa_destroy);
  2049. int img_mmu_vaa_alloc(struct mmu_vaa *vaa, size_t size, uint32_t *addr)
  2050. {
  2051. struct mem_man *mem_man = &mem_man_data;
  2052. struct imgmmu_halloc *alloc;
  2053. struct vaa_entry *entry;
  2054. int ret = 0;
  2055. if (!vaa || !addr || !size)
  2056. return -EINVAL;
  2057. entry = kzalloc(sizeof(struct vaa_entry), GFP_KERNEL);
  2058. if (!entry)
  2059. return -ENOMEM;
  2060. mutex_lock(&mem_man->mutex);
  2061. alloc = imgmmu_hallocate(vaa->heap, size, &ret);
  2062. if (!alloc || ret) {
  2063. pr_err("%s: imgmmu_hallocate failed (%zu)!\n",
  2064. __func__, size);
  2065. kfree(entry);
  2066. ret = -EFAULT;
  2067. goto exit;
  2068. }
  2069. entry->alloc = alloc;
  2070. list_add(&entry->mmu_vaa_entry, &vaa->entries);
  2071. *addr = alloc->vaddr;
  2072. exit:
  2073. mutex_unlock(&mem_man->mutex);
  2074. return ret;
  2075. }
  2076. EXPORT_SYMBOL(img_mmu_vaa_alloc);
  2077. int img_mmu_vaa_free(struct mmu_vaa *vaa, uint32_t addr, size_t size)
  2078. {
  2079. struct mem_man *mem_man = &mem_man_data;
  2080. struct vaa_entry *entry;
  2081. int ret = 0;
  2082. if (!vaa || !size)
  2083. return -EINVAL;
  2084. mutex_lock(&mem_man->mutex);
  2085. entry = list_first_entry(&vaa->entries,
  2086. struct vaa_entry, mmu_vaa_entry);
  2087. while (!entry) {
  2088. if (entry->alloc->vaddr == addr &&
  2089. entry->alloc->size == size)
  2090. break;
  2091. /* advance */
  2092. entry = list_next_entry(entry, mmu_vaa_entry);
  2093. }
  2094. if (!entry) {
  2095. pr_err("%s: allocation not found (0x%x:%zu)!\n",
  2096. __func__, addr, size);
  2097. ret = -EINVAL;
  2098. goto exit;
  2099. }
  2100. if (imgmmu_hfree(entry->alloc)) {
  2101. pr_err("%s: imgmmu_hfree failed (0x%x:%zu)!\n",
  2102. __func__, addr, size);
  2103. ret = -EFAULT;
  2104. goto exit;
  2105. }
  2106. list_del(&entry->mmu_vaa_entry);
  2107. kfree(entry);
  2108. exit:
  2109. mutex_unlock(&mem_man->mutex);
  2110. return ret;
  2111. }
  2112. EXPORT_SYMBOL(img_mmu_vaa_free);
  2113. // Parity look-up table for 8bits
  2114. static unsigned int _parity_lut_[256] =
  2115. {0, 1, 1, 0, 1, 0, 0, 1,
  2116. 1, 0, 0, 1, 0, 1, 1, 0,
  2117. 1, 0, 0, 1, 0, 1, 1, 0,
  2118. 0, 1, 1, 0, 1, 0, 0, 1,
  2119. 1, 0, 0, 1, 0, 1, 1, 0,
  2120. 0, 1, 1, 0, 1, 0, 0, 1,
  2121. 0, 1, 1, 0, 1, 0, 0, 1,
  2122. 1, 0, 0, 1, 0, 1, 1, 0,
  2123. 1, 0, 0, 1, 0, 1, 1, 0,
  2124. 0, 1, 1, 0, 1, 0, 0, 1,
  2125. 0, 1, 1, 0, 1, 0, 0, 1,
  2126. 1, 0, 0, 1, 0, 1, 1, 0,
  2127. 0, 1, 1, 0, 1, 0, 0, 1,
  2128. 1, 0, 0, 1, 0, 1, 1, 0,
  2129. 1, 0, 0, 1, 0, 1, 1, 0,
  2130. 0, 1, 1, 0, 1, 0, 0, 1,
  2131. 1, 0, 0, 1, 0, 1, 1, 0,
  2132. 0, 1, 1, 0, 1, 0, 0, 1,
  2133. 0, 1, 1, 0, 1, 0, 0, 1,
  2134. 1, 0, 0, 1, 0, 1, 1, 0,
  2135. 0, 1, 1, 0, 1, 0, 0, 1,
  2136. 1, 0, 0, 1, 0, 1, 1, 0,
  2137. 1, 0, 0, 1, 0, 1, 1, 0,
  2138. 0, 1, 1, 0, 1, 0, 0, 1,
  2139. 0, 1, 1, 0, 1, 0, 0, 1,
  2140. 1, 0, 0, 1, 0, 1, 1, 0,
  2141. 1, 0, 0, 1, 0, 1, 1, 0,
  2142. 0, 1, 1, 0, 1, 0, 0, 1,
  2143. 1, 0, 0, 1, 0, 1, 1, 0,
  2144. 0, 1, 1, 0, 1, 0, 0, 1,
  2145. 0, 1, 1, 0, 1, 0, 0, 1,
  2146. 1, 0, 0, 1, 0, 1, 1, 0};
  2147. bool img_mem_calc_parity(unsigned long long input)
  2148. {
  2149. // Split by half as number is considered to be of 64 bits
  2150. int bits;
  2151. // Dividing the number into 8-bit chunks while performing X-OR
  2152. for (bits = 32; bits >= 8; bits>>=1)
  2153. input = input ^ (input >> bits);
  2154. return _parity_lut_[input & 0xff] ? true : false;
  2155. }
  2156. EXPORT_SYMBOL(img_mem_calc_parity);
  2157. /*
  2158. * Initialisation
  2159. */
  2160. static int __init img_mem_init(void)
  2161. {
  2162. struct mem_man *mem_man = &mem_man_data;
  2163. pr_debug("%s:%d\n", __func__, __LINE__);
  2164. idr_init(&mem_man->heaps);
  2165. idr_init(&mem_man->mem_ctxs);
  2166. mutex_init(&mem_man->mutex);
  2167. mem_man->cache_usage = 0;
  2168. return 0;
  2169. }
  2170. static void __exit img_mem_exit(void)
  2171. {
  2172. struct mem_man *mem_man = &mem_man_data;
  2173. struct heap *heap;
  2174. struct mem_ctx *ctx;
  2175. int heap_id;
  2176. int ctx_id;
  2177. pr_debug("%s:%d\n", __func__, __LINE__);
  2178. /* keeps mutex checks (WARN_ON) happy, this will never actually wait */
  2179. mutex_lock(&mem_man->mutex);
  2180. ctx_id = 0;
  2181. ctx = idr_get_next(&mem_man->mem_ctxs, &ctx_id);
  2182. while (ctx) {
  2183. pr_warn("%s derelict memory context %p!\n", __func__, ctx);
  2184. _img_mem_destroy_proc_ctx(ctx);
  2185. kfree(ctx);
  2186. ctx_id = 0;
  2187. ctx = idr_get_next(&mem_man->mem_ctxs, &ctx_id);
  2188. }
  2189. heap_id = IMG_MEM_MAN_MIN_HEAP;
  2190. heap = idr_get_next(&mem_man->heaps, &heap_id);
  2191. while (heap) {
  2192. pr_warn("%s derelict heap %d!\n", __func__, heap_id);
  2193. _img_mem_del_heap(heap);
  2194. kfree(heap);
  2195. heap_id = IMG_MEM_MAN_MIN_HEAP;
  2196. heap = idr_get_next(&mem_man->heaps, &heap_id);
  2197. }
  2198. idr_destroy(&mem_man->heaps);
  2199. idr_destroy(&mem_man->mem_ctxs);
  2200. mutex_unlock(&mem_man->mutex);
  2201. mutex_destroy(&mem_man->mutex);
  2202. }
  2203. module_init(img_mem_init);
  2204. module_exit(img_mem_exit);
  2205. MODULE_LICENSE("GPL");
  2206. /*
  2207. * coding style for emacs
  2208. *
  2209. * Local variables:
  2210. * indent-tabs-mode: t
  2211. * tab-width: 8
  2212. * c-basic-offset: 8
  2213. * End:
  2214. */