cvmx-bootmem.c 41 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2018-2020 Marvell International Ltd.
  4. */
  5. /*
  6. * Simple allocate only memory allocator. Used to allocate memory at
  7. * application start time.
  8. */
  9. #include <asm/global_data.h>
  10. #include <linux/compat.h>
  11. #include <linux/io.h>
  12. #include <linux/types.h>
  13. #include <mach/octeon-model.h>
  14. #include <mach/cvmx-bootmem.h>
  15. #include <mach/cvmx-coremask.h>
  16. #include <mach/cvmx-regs.h>
  17. DECLARE_GLOBAL_DATA_PTR;
  18. /**
  19. * This is the physical location of a struct cvmx_bootmem_desc
  20. * structure in Octeon's memory. Note that dues to addressing
  21. * limits or runtime environment it might not be possible to
  22. * create a C pointer to this structure.
  23. */
  24. static u64 cvmx_bootmem_desc_addr;
  25. /**
  26. * This macro returns the size of a member of a structure.
  27. * Logically it is the same as "sizeof(s::field)" in C++, but
  28. * C lacks the "::" operator.
  29. */
  30. #define SIZEOF_FIELD(s, field) sizeof(((s *)NULL)->field)
  31. /**
  32. * This macro returns a member of the struct cvmx_bootmem_desc
  33. * structure. These members can't be directly addressed as
  34. * they might be in memory not directly reachable. In the case
  35. * where bootmem is compiled with LINUX_HOST, the structure
  36. * itself might be located on a remote Octeon. The argument
  37. * "field" is the member name of the struct cvmx_bootmem_desc to read.
  38. * Regardless of the type of the field, the return type is always
  39. * a u64.
  40. */
  41. #define CVMX_BOOTMEM_DESC_GET_FIELD(field) \
  42. __cvmx_bootmem_desc_get(cvmx_bootmem_desc_addr, \
  43. offsetof(struct cvmx_bootmem_desc, field), \
  44. SIZEOF_FIELD(struct cvmx_bootmem_desc, field))
  45. /**
  46. * This macro writes a member of the struct cvmx_bootmem_desc
  47. * structure. These members can't be directly addressed as
  48. * they might be in memory not directly reachable. In the case
  49. * where bootmem is compiled with LINUX_HOST, the structure
  50. * itself might be located on a remote Octeon. The argument
  51. * "field" is the member name of the struct cvmx_bootmem_desc to write.
  52. */
  53. #define CVMX_BOOTMEM_DESC_SET_FIELD(field, value) \
  54. __cvmx_bootmem_desc_set(cvmx_bootmem_desc_addr, \
  55. offsetof(struct cvmx_bootmem_desc, field), \
  56. SIZEOF_FIELD(struct cvmx_bootmem_desc, field), \
  57. value)
  58. /**
  59. * This macro returns a member of the
  60. * struct cvmx_bootmem_named_block_desc structure. These members can't
  61. * be directly addressed as they might be in memory not directly
  62. * reachable. In the case where bootmem is compiled with
  63. * LINUX_HOST, the structure itself might be located on a remote
  64. * Octeon. The argument "field" is the member name of the
  65. * struct cvmx_bootmem_named_block_desc to read. Regardless of the type
  66. * of the field, the return type is always a u64. The "addr"
  67. * parameter is the physical address of the structure.
  68. */
  69. #define CVMX_BOOTMEM_NAMED_GET_FIELD(addr, field) \
  70. __cvmx_bootmem_desc_get(addr, \
  71. offsetof(struct cvmx_bootmem_named_block_desc, field), \
  72. SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field))
  73. /**
  74. * This macro writes a member of the struct cvmx_bootmem_named_block_desc
  75. * structure. These members can't be directly addressed as
  76. * they might be in memory not directly reachable. In the case
  77. * where bootmem is compiled with LINUX_HOST, the structure
  78. * itself might be located on a remote Octeon. The argument
  79. * "field" is the member name of the
  80. * struct cvmx_bootmem_named_block_desc to write. The "addr" parameter
  81. * is the physical address of the structure.
  82. */
  83. #define CVMX_BOOTMEM_NAMED_SET_FIELD(addr, field, value) \
  84. __cvmx_bootmem_desc_set(addr, \
  85. offsetof(struct cvmx_bootmem_named_block_desc, field), \
  86. SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field), \
  87. value)
  88. /**
  89. * This function is the implementation of the get macros defined
  90. * for individual structure members. The argument are generated
  91. * by the macros inorder to read only the needed memory.
  92. *
  93. * @param base 64bit physical address of the complete structure
  94. * @param offset Offset from the beginning of the structure to the member being
  95. * accessed.
  96. * @param size Size of the structure member.
  97. *
  98. * Return: Value of the structure member promoted into a u64.
  99. */
  100. static inline u64 __cvmx_bootmem_desc_get(u64 base, int offset,
  101. int size)
  102. {
  103. base = (1ull << 63) | (base + offset);
  104. switch (size) {
  105. case 4:
  106. return cvmx_read64_uint32(base);
  107. case 8:
  108. return cvmx_read64_uint64(base);
  109. default:
  110. return 0;
  111. }
  112. }
  113. /**
  114. * This function is the implementation of the set macros defined
  115. * for individual structure members. The argument are generated
  116. * by the macros in order to write only the needed memory.
  117. *
  118. * @param base 64bit physical address of the complete structure
  119. * @param offset Offset from the beginning of the structure to the member being
  120. * accessed.
  121. * @param size Size of the structure member.
  122. * @param value Value to write into the structure
  123. */
  124. static inline void __cvmx_bootmem_desc_set(u64 base, int offset, int size,
  125. u64 value)
  126. {
  127. base = (1ull << 63) | (base + offset);
  128. switch (size) {
  129. case 4:
  130. cvmx_write64_uint32(base, value);
  131. break;
  132. case 8:
  133. cvmx_write64_uint64(base, value);
  134. break;
  135. default:
  136. break;
  137. }
  138. }
  139. /**
  140. * This function returns the address of the bootmem descriptor lock.
  141. *
  142. * Return: 64-bit address in KSEG0 of the bootmem descriptor block
  143. */
  144. static inline u64 __cvmx_bootmem_get_lock_addr(void)
  145. {
  146. return (1ull << 63) |
  147. (cvmx_bootmem_desc_addr + offsetof(struct cvmx_bootmem_desc, lock));
  148. }
  149. /**
  150. * This function retrieves the string name of a named block. It is
  151. * more complicated than a simple memcpy() since the named block
  152. * descriptor may not be directly accessible.
  153. *
  154. * @param addr Physical address of the named block descriptor
  155. * @param str String to receive the named block string name
  156. * @param len Length of the string buffer, which must match the length
  157. * stored in the bootmem descriptor.
  158. */
  159. static void CVMX_BOOTMEM_NAMED_GET_NAME(u64 addr, char *str, int len)
  160. {
  161. int l = len;
  162. char *ptr = str;
  163. addr |= (1ull << 63);
  164. addr += offsetof(struct cvmx_bootmem_named_block_desc, name);
  165. while (l) {
  166. /*
  167. * With big-endian in memory byte order, this gives uniform
  168. * results for the CPU in either big or Little endian mode.
  169. */
  170. u64 blob = cvmx_read64_uint64(addr);
  171. int sa = 56;
  172. addr += sizeof(u64);
  173. while (l && sa >= 0) {
  174. *ptr++ = (char)(blob >> sa);
  175. l--;
  176. sa -= 8;
  177. }
  178. }
  179. str[len] = 0;
  180. }
  181. /**
  182. * This function stores the string name of a named block. It is
  183. * more complicated than a simple memcpy() since the named block
  184. * descriptor may not be directly accessible.
  185. *
  186. * @param addr Physical address of the named block descriptor
  187. * @param str String to store into the named block string name
  188. * @param len Length of the string buffer, which must match the length
  189. * stored in the bootmem descriptor.
  190. */
  191. void CVMX_BOOTMEM_NAMED_SET_NAME(u64 addr, const char *str, int len)
  192. {
  193. int l = len;
  194. addr |= (1ull << 63);
  195. addr += offsetof(struct cvmx_bootmem_named_block_desc, name);
  196. while (l) {
  197. /*
  198. * With big-endian in memory byte order, this gives uniform
  199. * results for the CPU in either big or Little endian mode.
  200. */
  201. u64 blob = 0;
  202. int sa = 56;
  203. while (l && sa >= 0) {
  204. u64 c = (u8)(*str++);
  205. l--;
  206. if (l == 0)
  207. c = 0;
  208. blob |= c << sa;
  209. sa -= 8;
  210. }
  211. cvmx_write64_uint64(addr, blob);
  212. addr += sizeof(u64);
  213. }
  214. }
  215. /* See header file for descriptions of functions */
  216. /*
  217. * Wrapper functions are provided for reading/writing the size and next block
  218. * values as these may not be directly addressible (in 32 bit applications, for
  219. * instance.)
  220. *
  221. * Offsets of data elements in bootmem list, must match
  222. * struct cvmx_bootmem_block_header
  223. */
  224. #define NEXT_OFFSET 0
  225. #define SIZE_OFFSET 8
  226. static void cvmx_bootmem_phy_set_size(u64 addr, u64 size)
  227. {
  228. cvmx_write64_uint64((addr + SIZE_OFFSET) | (1ull << 63), size);
  229. }
  230. static void cvmx_bootmem_phy_set_next(u64 addr, u64 next)
  231. {
  232. cvmx_write64_uint64((addr + NEXT_OFFSET) | (1ull << 63), next);
  233. }
  234. static u64 cvmx_bootmem_phy_get_size(u64 addr)
  235. {
  236. return cvmx_read64_uint64((addr + SIZE_OFFSET) | (1ull << 63));
  237. }
  238. static u64 cvmx_bootmem_phy_get_next(u64 addr)
  239. {
  240. return cvmx_read64_uint64((addr + NEXT_OFFSET) | (1ull << 63));
  241. }
  242. /**
  243. * Check the version information on the bootmem descriptor
  244. *
  245. * @param exact_match
  246. * Exact major version to check against. A zero means
  247. * check that the version supports named blocks.
  248. *
  249. * Return: Zero if the version is correct. Negative if the version is
  250. * incorrect. Failures also cause a message to be displayed.
  251. */
  252. static int __cvmx_bootmem_check_version(int exact_match)
  253. {
  254. int major_version;
  255. major_version = CVMX_BOOTMEM_DESC_GET_FIELD(major_version);
  256. if ((major_version > 3) ||
  257. (exact_match && major_version != exact_match)) {
  258. debug("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: 0x%llx\n",
  259. major_version,
  260. (int)CVMX_BOOTMEM_DESC_GET_FIELD(minor_version),
  261. CAST_ULL(cvmx_bootmem_desc_addr));
  262. return -1;
  263. } else {
  264. return 0;
  265. }
  266. }
  267. /**
  268. * Get the low level bootmem descriptor lock. If no locking
  269. * is specified in the flags, then nothing is done.
  270. *
  271. * @param flags CVMX_BOOTMEM_FLAG_NO_LOCKING means this functions should do
  272. * nothing. This is used to support nested bootmem calls.
  273. */
  274. static inline void __cvmx_bootmem_lock(u32 flags)
  275. {
  276. if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING)) {
  277. /*
  278. * Unfortunately we can't use the normal cvmx-spinlock code as
  279. * the memory for the bootmem descriptor may be not accessible
  280. * by a C pointer. We use a 64bit XKPHYS address to access the
  281. * memory directly
  282. */
  283. u64 lock_addr = (1ull << 63) |
  284. (cvmx_bootmem_desc_addr + offsetof(struct cvmx_bootmem_desc,
  285. lock));
  286. unsigned int tmp;
  287. __asm__ __volatile__(".set noreorder\n"
  288. "1: ll %[tmp], 0(%[addr])\n"
  289. " bnez %[tmp], 1b\n"
  290. " li %[tmp], 1\n"
  291. " sc %[tmp], 0(%[addr])\n"
  292. " beqz %[tmp], 1b\n"
  293. " nop\n"
  294. ".set reorder\n"
  295. : [tmp] "=&r"(tmp)
  296. : [addr] "r"(lock_addr)
  297. : "memory");
  298. }
  299. }
  300. /**
  301. * Release the low level bootmem descriptor lock. If no locking
  302. * is specified in the flags, then nothing is done.
  303. *
  304. * @param flags CVMX_BOOTMEM_FLAG_NO_LOCKING means this functions should do
  305. * nothing. This is used to support nested bootmem calls.
  306. */
  307. static inline void __cvmx_bootmem_unlock(u32 flags)
  308. {
  309. if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING)) {
  310. /*
  311. * Unfortunately we can't use the normal cvmx-spinlock code as
  312. * the memory for the bootmem descriptor may be not accessible
  313. * by a C pointer. We use a 64bit XKPHYS address to access the
  314. * memory directly
  315. */
  316. u64 lock_addr = __cvmx_bootmem_get_lock_addr();
  317. CVMX_SYNCW;
  318. __asm__ __volatile__("sw $0, 0(%[addr])\n"
  319. : : [addr] "r"(lock_addr)
  320. : "memory");
  321. CVMX_SYNCW;
  322. }
  323. }
  324. /*
  325. * Some of the cvmx-bootmem functions dealing with C pointers are not
  326. * supported when we are compiling for CVMX_BUILD_FOR_LINUX_HOST. This
  327. * ifndef removes these functions when they aren't needed.
  328. *
  329. * This functions takes an address range and adjusts it as necessary
  330. * to match the ABI that is currently being used. This is required to
  331. * ensure that bootmem_alloc* functions only return valid pointers for
  332. * 32 bit ABIs
  333. */
  334. static int __cvmx_validate_mem_range(u64 *min_addr_ptr,
  335. u64 *max_addr_ptr)
  336. {
  337. u64 max_phys = (1ull << 29) - 0x10; /* KSEG0 */
  338. *min_addr_ptr = min_t(u64, max_t(u64, *min_addr_ptr, 0x0), max_phys);
  339. if (!*max_addr_ptr) {
  340. *max_addr_ptr = max_phys;
  341. } else {
  342. *max_addr_ptr = max_t(u64, min_t(u64, *max_addr_ptr,
  343. max_phys), 0x0);
  344. }
  345. return 0;
  346. }
  347. u64 cvmx_bootmem_phy_alloc_range(u64 size, u64 alignment,
  348. u64 min_addr, u64 max_addr)
  349. {
  350. s64 address;
  351. __cvmx_validate_mem_range(&min_addr, &max_addr);
  352. address = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
  353. alignment, 0);
  354. if (address > 0)
  355. return address;
  356. else
  357. return 0;
  358. }
  359. void *cvmx_bootmem_alloc_range(u64 size, u64 alignment,
  360. u64 min_addr, u64 max_addr)
  361. {
  362. s64 address;
  363. __cvmx_validate_mem_range(&min_addr, &max_addr);
  364. address = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
  365. alignment, 0);
  366. if (address > 0)
  367. return cvmx_phys_to_ptr(address);
  368. else
  369. return NULL;
  370. }
  371. void *cvmx_bootmem_alloc_address(u64 size, u64 address,
  372. u64 alignment)
  373. {
  374. return cvmx_bootmem_alloc_range(size, alignment, address,
  375. address + size);
  376. }
  377. void *cvmx_bootmem_alloc_node(u64 node, u64 size, u64 alignment)
  378. {
  379. return cvmx_bootmem_alloc_range(size, alignment,
  380. node << CVMX_NODE_MEM_SHIFT,
  381. ((node + 1) << CVMX_NODE_MEM_SHIFT) - 1);
  382. }
  383. void *cvmx_bootmem_alloc(u64 size, u64 alignment)
  384. {
  385. return cvmx_bootmem_alloc_range(size, alignment, 0, 0);
  386. }
  387. void *cvmx_bootmem_alloc_named_range_once(u64 size, u64 min_addr,
  388. u64 max_addr, u64 align,
  389. const char *name,
  390. void (*init)(void *))
  391. {
  392. u64 named_block_desc_addr;
  393. void *ptr;
  394. s64 addr;
  395. __cvmx_bootmem_lock(0);
  396. __cvmx_validate_mem_range(&min_addr, &max_addr);
  397. named_block_desc_addr =
  398. cvmx_bootmem_phy_named_block_find(name,
  399. CVMX_BOOTMEM_FLAG_NO_LOCKING);
  400. if (named_block_desc_addr) {
  401. addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_desc_addr,
  402. base_addr);
  403. __cvmx_bootmem_unlock(0);
  404. return cvmx_phys_to_ptr(addr);
  405. }
  406. addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
  407. align, name,
  408. CVMX_BOOTMEM_FLAG_NO_LOCKING);
  409. if (addr < 0) {
  410. __cvmx_bootmem_unlock(0);
  411. return NULL;
  412. }
  413. ptr = cvmx_phys_to_ptr(addr);
  414. if (init)
  415. init(ptr);
  416. else
  417. memset(ptr, 0, size);
  418. __cvmx_bootmem_unlock(0);
  419. return ptr;
  420. }
  421. void *cvmx_bootmem_alloc_named_range_flags(u64 size, u64 min_addr,
  422. u64 max_addr, u64 align,
  423. const char *name, u32 flags)
  424. {
  425. s64 addr;
  426. __cvmx_validate_mem_range(&min_addr, &max_addr);
  427. addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
  428. align, name, flags);
  429. if (addr >= 0)
  430. return cvmx_phys_to_ptr(addr);
  431. else
  432. return NULL;
  433. }
  434. void *cvmx_bootmem_alloc_named_range(u64 size, u64 min_addr,
  435. u64 max_addr, u64 align,
  436. const char *name)
  437. {
  438. return cvmx_bootmem_alloc_named_range_flags(size, min_addr, max_addr,
  439. align, name, 0);
  440. }
  441. void *cvmx_bootmem_alloc_named_address(u64 size, u64 address,
  442. const char *name)
  443. {
  444. return cvmx_bootmem_alloc_named_range(size, address, address + size,
  445. 0, name);
  446. }
  447. void *cvmx_bootmem_alloc_named(u64 size, u64 alignment,
  448. const char *name)
  449. {
  450. return cvmx_bootmem_alloc_named_range(size, 0, 0, alignment, name);
  451. }
  452. void *cvmx_bootmem_alloc_named_flags(u64 size, u64 alignment,
  453. const char *name, u32 flags)
  454. {
  455. return cvmx_bootmem_alloc_named_range_flags(size, 0, 0, alignment,
  456. name, flags);
  457. }
  458. int cvmx_bootmem_free_named(const char *name)
  459. {
  460. return cvmx_bootmem_phy_named_block_free(name, 0);
  461. }
  462. /**
  463. * Find a named block with flags
  464. *
  465. * @param name is the block name
  466. * @param flags indicates the need to use locking during search
  467. * Return: pointer to named block descriptor
  468. *
  469. * Note: this function returns a pointer to a static structure,
  470. * and is therefore not re-entrant.
  471. * Making this function re-entrant will break backward compatibility.
  472. */
  473. const struct cvmx_bootmem_named_block_desc *
  474. __cvmx_bootmem_find_named_block_flags(const char *name, u32 flags)
  475. {
  476. static struct cvmx_bootmem_named_block_desc desc;
  477. u64 named_addr = cvmx_bootmem_phy_named_block_find(name, flags);
  478. if (named_addr) {
  479. desc.base_addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr,
  480. base_addr);
  481. desc.size = CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr, size);
  482. strncpy(desc.name, name, sizeof(desc.name));
  483. desc.name[sizeof(desc.name) - 1] = 0;
  484. return &desc;
  485. } else {
  486. return NULL;
  487. }
  488. }
  489. const struct cvmx_bootmem_named_block_desc *
  490. cvmx_bootmem_find_named_block(const char *name)
  491. {
  492. return __cvmx_bootmem_find_named_block_flags(name, 0);
  493. }
  494. void cvmx_bootmem_print_named(void)
  495. {
  496. cvmx_bootmem_phy_named_block_print();
  497. }
  498. int cvmx_bootmem_init(u64 mem_desc_addr)
  499. {
  500. if (!cvmx_bootmem_desc_addr)
  501. cvmx_bootmem_desc_addr = mem_desc_addr;
  502. return 0;
  503. }
  504. u64 cvmx_bootmem_available_mem(u64 min_block_size)
  505. {
  506. return cvmx_bootmem_phy_available_mem(min_block_size);
  507. }
  508. /*
  509. * The cvmx_bootmem_phy* functions below return 64 bit physical
  510. * addresses, and expose more features that the cvmx_bootmem_functions
  511. * above. These are required for full memory space access in 32 bit
  512. * applications, as well as for using some advance features. Most
  513. * applications should not need to use these.
  514. */
  515. s64 cvmx_bootmem_phy_alloc(u64 req_size, u64 address_min,
  516. u64 address_max, u64 alignment,
  517. u32 flags)
  518. {
  519. u64 head_addr, ent_addr, ent_size;
  520. u64 target_ent_addr = 0, target_prev_addr = 0;
  521. u64 target_size = ~0ull;
  522. u64 free_start, free_end;
  523. u64 next_addr, prev_addr = 0;
  524. u64 new_ent_addr = 0, new_ent_size;
  525. u64 desired_min_addr, usable_max;
  526. u64 align, align_mask;
  527. debug("%s: req_size: 0x%llx, min_addr: 0x%llx, max_addr: 0x%llx, align: 0x%llx\n",
  528. __func__, CAST_ULL(req_size), CAST_ULL(address_min),
  529. CAST_ULL(address_max), CAST_ULL(alignment));
  530. if (__cvmx_bootmem_check_version(0))
  531. return -1;
  532. /*
  533. * Do a variety of checks to validate the arguments. The
  534. * allocator code will later assume that these checks have
  535. * been made. We validate that the requested constraints are
  536. * not self-contradictory before we look through the list of
  537. * available memory
  538. */
  539. /* 0 is not a valid req_size for this allocator */
  540. if (!req_size)
  541. return -1;
  542. /* Round req_size up to multiple of minimum alignment bytes */
  543. req_size = (req_size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
  544. ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
  545. /* Make sure alignment is power of 2, and at least the minimum */
  546. for (align = CVMX_BOOTMEM_ALIGNMENT_SIZE;
  547. align < (1ull << 48);
  548. align <<= 1) {
  549. if (align >= alignment)
  550. break;
  551. }
  552. align_mask = ~(align - 1);
  553. /*
  554. * Adjust address minimum based on requested alignment (round
  555. * up to meet alignment). Do this here so we can reject
  556. * impossible requests up front. (NOP for address_min == 0)
  557. */
  558. address_min = (address_min + (align - 1)) & align_mask;
  559. /*
  560. * Convert !0 address_min and 0 address_max to special case of
  561. * range that specifies an exact memory block to allocate. Do
  562. * this before other checks and adjustments so that this
  563. * tranformation will be validated
  564. */
  565. if (address_min && !address_max)
  566. address_max = address_min + req_size;
  567. else if (!address_min && !address_max)
  568. address_max = ~0ull; /* If no limits given, use max */
  569. /*
  570. * Reject inconsistent args. We have adjusted these, so this
  571. * may fail due to our internal changes even if this check
  572. * would pass for the values the user supplied.
  573. */
  574. if (req_size > address_max - address_min)
  575. return -1;
  576. __cvmx_bootmem_lock(flags);
  577. /* Walk through the list entries to find the right fit */
  578. head_addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
  579. for (ent_addr = head_addr;
  580. ent_addr != 0ULL && ent_addr < address_max;
  581. prev_addr = ent_addr,
  582. ent_addr = cvmx_bootmem_phy_get_next(ent_addr)) {
  583. /* Raw free block size */
  584. ent_size = cvmx_bootmem_phy_get_size(ent_addr);
  585. next_addr = cvmx_bootmem_phy_get_next(ent_addr);
  586. /* Validate the free list ascending order */
  587. if (ent_size < CVMX_BOOTMEM_ALIGNMENT_SIZE ||
  588. (next_addr && ent_addr > next_addr)) {
  589. debug("ERROR: %s: bad free list ent: %#llx, next: %#llx\n",
  590. __func__, CAST_ULL(ent_addr),
  591. CAST_ULL(next_addr));
  592. goto error_out;
  593. }
  594. /* adjust free block edges for alignment */
  595. free_start = (ent_addr + align - 1) & align_mask;
  596. free_end = (ent_addr + ent_size) & align_mask;
  597. /* check that free block is large enough */
  598. if ((free_start + req_size) > free_end)
  599. continue;
  600. /* check that desired start is within the free block */
  601. if (free_end < address_min || free_start > address_max)
  602. continue;
  603. if ((free_end - address_min) < req_size)
  604. continue;
  605. if ((address_max - free_start) < req_size)
  606. continue;
  607. /* Found usebale free block */
  608. target_ent_addr = ent_addr;
  609. target_prev_addr = prev_addr;
  610. target_size = ent_size;
  611. /* Continue looking for highest/best block that fits */
  612. }
  613. /* Bail if the search has resulted in no eligible free blocks */
  614. if (target_ent_addr == 0) {
  615. debug("%s: eligible free block not found\n", __func__);
  616. goto error_out;
  617. }
  618. /* Found the free block to allocate from */
  619. ent_addr = target_ent_addr;
  620. prev_addr = target_prev_addr;
  621. ent_size = target_size;
  622. debug("%s: using free block at %#010llx size %#llx\n",
  623. __func__, CAST_ULL(ent_addr), CAST_ULL(ent_size));
  624. /* Always allocate from the end of a free block */
  625. usable_max = min_t(u64, address_max, ent_addr + ent_size);
  626. desired_min_addr = usable_max - req_size;
  627. desired_min_addr &= align_mask;
  628. /* Split current free block into up to 3 free blocks */
  629. /* Check for head room */
  630. if (desired_min_addr > ent_addr) {
  631. /* Create a new free block at the allocation address */
  632. new_ent_addr = desired_min_addr;
  633. new_ent_size = ent_size - (desired_min_addr - ent_addr);
  634. cvmx_bootmem_phy_set_next(new_ent_addr,
  635. cvmx_bootmem_phy_get_next(ent_addr));
  636. cvmx_bootmem_phy_set_size(new_ent_addr, new_ent_size);
  637. /* Split out head room into a new free block */
  638. ent_size -= new_ent_size;
  639. cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
  640. cvmx_bootmem_phy_set_size(ent_addr, ent_size);
  641. debug("%s: splitting head, addr %#llx size %#llx\n",
  642. __func__, CAST_ULL(ent_addr), CAST_ULL(ent_size));
  643. /* Make the allocation target the current free block */
  644. prev_addr = ent_addr;
  645. ent_addr = new_ent_addr;
  646. ent_size = new_ent_size;
  647. }
  648. /* Check for tail room */
  649. if ((desired_min_addr + req_size) < (ent_addr + ent_size)) {
  650. new_ent_addr = ent_addr + req_size;
  651. new_ent_size = ent_size - req_size;
  652. /* Create a new free block from tail room */
  653. cvmx_bootmem_phy_set_next(new_ent_addr,
  654. cvmx_bootmem_phy_get_next(ent_addr));
  655. cvmx_bootmem_phy_set_size(new_ent_addr, new_ent_size);
  656. debug("%s: splitting tail, addr %#llx size %#llx\n",
  657. __func__, CAST_ULL(new_ent_addr), CAST_ULL(new_ent_size));
  658. /* Adjust the current block to exclude tail room */
  659. ent_size = ent_size - new_ent_size;
  660. cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
  661. cvmx_bootmem_phy_set_size(ent_addr, ent_size);
  662. }
  663. /* The current free block IS the allocation target */
  664. if (desired_min_addr != ent_addr || ent_size != req_size)
  665. debug("ERROR: %s: internal error - addr %#llx %#llx size %#llx %#llx\n",
  666. __func__, CAST_ULL(desired_min_addr), CAST_ULL(ent_addr),
  667. CAST_ULL(ent_size), CAST_ULL(req_size));
  668. /* Remove the current free block from list */
  669. if (prev_addr) {
  670. cvmx_bootmem_phy_set_next(prev_addr,
  671. cvmx_bootmem_phy_get_next(ent_addr));
  672. } else {
  673. /* head of list being returned, so update head ptr */
  674. CVMX_BOOTMEM_DESC_SET_FIELD(head_addr,
  675. cvmx_bootmem_phy_get_next(ent_addr));
  676. }
  677. __cvmx_bootmem_unlock(flags);
  678. debug("%s: allocated size: %#llx, at addr: %#010llx\n",
  679. __func__,
  680. CAST_ULL(req_size),
  681. CAST_ULL(desired_min_addr));
  682. return desired_min_addr;
  683. error_out:
  684. /* Requested memory not found or argument error */
  685. __cvmx_bootmem_unlock(flags);
  686. return -1;
  687. }
  688. int __cvmx_bootmem_phy_free(u64 phy_addr, u64 size, u32 flags)
  689. {
  690. u64 cur_addr;
  691. u64 prev_addr = 0; /* zero is invalid */
  692. int retval = 0;
  693. debug("%s addr: %#llx, size: %#llx\n", __func__,
  694. CAST_ULL(phy_addr), CAST_ULL(size));
  695. if (__cvmx_bootmem_check_version(0))
  696. return 0;
  697. /* 0 is not a valid size for this allocator */
  698. if (!size || !phy_addr)
  699. return 0;
  700. /* Round size up to mult of minimum alignment bytes */
  701. size = (size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
  702. ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
  703. __cvmx_bootmem_lock(flags);
  704. cur_addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
  705. if (cur_addr == 0 || phy_addr < cur_addr) {
  706. /* add at front of list - special case with changing head ptr */
  707. if (cur_addr && phy_addr + size > cur_addr)
  708. goto bootmem_free_done; /* error, overlapping section */
  709. else if (phy_addr + size == cur_addr) {
  710. /* Add to front of existing first block */
  711. cvmx_bootmem_phy_set_next(phy_addr,
  712. cvmx_bootmem_phy_get_next(cur_addr));
  713. cvmx_bootmem_phy_set_size(phy_addr,
  714. cvmx_bootmem_phy_get_size(cur_addr) + size);
  715. CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, phy_addr);
  716. } else {
  717. /* New block before first block */
  718. /* OK if cur_addr is 0 */
  719. cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
  720. cvmx_bootmem_phy_set_size(phy_addr, size);
  721. CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, phy_addr);
  722. }
  723. retval = 1;
  724. goto bootmem_free_done;
  725. }
  726. /* Find place in list to add block */
  727. while (cur_addr && phy_addr > cur_addr) {
  728. prev_addr = cur_addr;
  729. cur_addr = cvmx_bootmem_phy_get_next(cur_addr);
  730. }
  731. if (!cur_addr) {
  732. /*
  733. * We have reached the end of the list, add on to end, checking
  734. * to see if we need to combine with last block
  735. */
  736. if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) == phy_addr) {
  737. cvmx_bootmem_phy_set_size(prev_addr,
  738. cvmx_bootmem_phy_get_size(prev_addr) + size);
  739. } else {
  740. cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
  741. cvmx_bootmem_phy_set_size(phy_addr, size);
  742. cvmx_bootmem_phy_set_next(phy_addr, 0);
  743. }
  744. retval = 1;
  745. goto bootmem_free_done;
  746. } else {
  747. /*
  748. * insert between prev and cur nodes, checking for merge with
  749. * either/both
  750. */
  751. if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) == phy_addr) {
  752. /* Merge with previous */
  753. cvmx_bootmem_phy_set_size(prev_addr,
  754. cvmx_bootmem_phy_get_size(prev_addr) + size);
  755. if (phy_addr + size == cur_addr) {
  756. /* Also merge with current */
  757. cvmx_bootmem_phy_set_size(prev_addr,
  758. cvmx_bootmem_phy_get_size(cur_addr) +
  759. cvmx_bootmem_phy_get_size(prev_addr));
  760. cvmx_bootmem_phy_set_next(prev_addr,
  761. cvmx_bootmem_phy_get_next(cur_addr));
  762. }
  763. retval = 1;
  764. goto bootmem_free_done;
  765. } else if (phy_addr + size == cur_addr) {
  766. /* Merge with current */
  767. cvmx_bootmem_phy_set_size(phy_addr,
  768. cvmx_bootmem_phy_get_size(cur_addr) + size);
  769. cvmx_bootmem_phy_set_next(phy_addr,
  770. cvmx_bootmem_phy_get_next(cur_addr));
  771. cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
  772. retval = 1;
  773. goto bootmem_free_done;
  774. }
  775. /* It is a standalone block, add in between prev and cur */
  776. cvmx_bootmem_phy_set_size(phy_addr, size);
  777. cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
  778. cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
  779. }
  780. retval = 1;
  781. bootmem_free_done:
  782. __cvmx_bootmem_unlock(flags);
  783. return retval;
  784. }
  785. void cvmx_bootmem_phy_list_print(void)
  786. {
  787. u64 addr;
  788. addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
  789. printf("\n\n\nPrinting bootmem block list, descriptor: 0x%llx, head is 0x%llx\n",
  790. CAST_ULL(cvmx_bootmem_desc_addr), CAST_ULL(addr));
  791. printf("Descriptor version: %d.%d\n",
  792. (int)CVMX_BOOTMEM_DESC_GET_FIELD(major_version),
  793. (int)CVMX_BOOTMEM_DESC_GET_FIELD(minor_version));
  794. if (CVMX_BOOTMEM_DESC_GET_FIELD(major_version) > 3)
  795. debug("Warning: Bootmem descriptor version is newer than expected\n");
  796. if (!addr)
  797. printf("mem list is empty!\n");
  798. while (addr) {
  799. printf("Block address: 0x%08llx, size: 0x%08llx, next: 0x%08llx\n", CAST_ULL(addr),
  800. CAST_ULL(cvmx_bootmem_phy_get_size(addr)),
  801. CAST_ULL(cvmx_bootmem_phy_get_next(addr)));
  802. addr = cvmx_bootmem_phy_get_next(addr);
  803. }
  804. printf("\n\n");
  805. }
  806. u64 cvmx_bootmem_phy_available_mem(u64 min_block_size)
  807. {
  808. u64 addr;
  809. u64 available_mem = 0;
  810. __cvmx_bootmem_lock(0);
  811. addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
  812. while (addr) {
  813. if (cvmx_bootmem_phy_get_size(addr) >= min_block_size)
  814. available_mem += cvmx_bootmem_phy_get_size(addr);
  815. addr = cvmx_bootmem_phy_get_next(addr);
  816. }
  817. __cvmx_bootmem_unlock(0);
  818. return available_mem;
  819. }
  820. u64 cvmx_bootmem_phy_named_block_find(const char *name, u32 flags)
  821. {
  822. u64 result = 0;
  823. debug("%s: %s\n", __func__, name);
  824. __cvmx_bootmem_lock(flags);
  825. if (!__cvmx_bootmem_check_version(3)) {
  826. int i;
  827. u64 named_block_array_addr =
  828. CVMX_BOOTMEM_DESC_GET_FIELD(named_block_array_addr);
  829. int num_blocks =
  830. CVMX_BOOTMEM_DESC_GET_FIELD(named_block_num_blocks);
  831. int name_length =
  832. CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len);
  833. u64 named_addr = named_block_array_addr;
  834. for (i = 0; i < num_blocks; i++) {
  835. u64 named_size =
  836. CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr, size);
  837. if (name && named_size) {
  838. char name_tmp[name_length + 1];
  839. CVMX_BOOTMEM_NAMED_GET_NAME(named_addr,
  840. name_tmp,
  841. name_length);
  842. if (!strncmp(name, name_tmp, name_length)) {
  843. result = named_addr;
  844. break;
  845. }
  846. } else if (!name && !named_size) {
  847. result = named_addr;
  848. break;
  849. }
  850. named_addr +=
  851. sizeof(struct cvmx_bootmem_named_block_desc);
  852. }
  853. }
  854. __cvmx_bootmem_unlock(flags);
  855. return result;
  856. }
  857. int cvmx_bootmem_phy_named_block_free(const char *name, u32 flags)
  858. {
  859. u64 named_block_addr;
  860. if (__cvmx_bootmem_check_version(3))
  861. return 0;
  862. debug("%s: %s\n", __func__, name);
  863. /*
  864. * Take lock here, as name lookup/block free/name free need to be
  865. * atomic
  866. */
  867. __cvmx_bootmem_lock(flags);
  868. named_block_addr = cvmx_bootmem_phy_named_block_find(name,
  869. CVMX_BOOTMEM_FLAG_NO_LOCKING);
  870. if (named_block_addr) {
  871. u64 named_addr =
  872. CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr,
  873. base_addr);
  874. u64 named_size =
  875. CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr, size);
  876. debug("%s: %s, base: 0x%llx, size: 0x%llx\n",
  877. __func__, name, CAST_ULL(named_addr),
  878. CAST_ULL(named_size));
  879. __cvmx_bootmem_phy_free(named_addr, named_size,
  880. CVMX_BOOTMEM_FLAG_NO_LOCKING);
  881. /* Set size to zero to indicate block not used. */
  882. CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_addr, size, 0);
  883. }
  884. __cvmx_bootmem_unlock(flags);
  885. return !!named_block_addr; /* 0 on failure, 1 on success */
  886. }
  887. s64 cvmx_bootmem_phy_named_block_alloc(u64 size, u64 min_addr,
  888. u64 max_addr,
  889. u64 alignment, const char *name,
  890. u32 flags)
  891. {
  892. s64 addr_allocated;
  893. u64 named_block_desc_addr;
  894. debug("%s: size: 0x%llx, min: 0x%llx, max: 0x%llx, align: 0x%llx, name: %s\n",
  895. __func__, CAST_ULL(size), CAST_ULL(min_addr), CAST_ULL(max_addr),
  896. CAST_ULL(alignment), name);
  897. if (__cvmx_bootmem_check_version(3))
  898. return -1;
  899. /*
  900. * Take lock here, as name lookup/block alloc/name add need to be
  901. * atomic
  902. */
  903. __cvmx_bootmem_lock(flags);
  904. named_block_desc_addr =
  905. cvmx_bootmem_phy_named_block_find(name, flags |
  906. CVMX_BOOTMEM_FLAG_NO_LOCKING);
  907. if (named_block_desc_addr) {
  908. __cvmx_bootmem_unlock(flags);
  909. return -1;
  910. }
  911. /* Get pointer to first available named block descriptor */
  912. named_block_desc_addr =
  913. cvmx_bootmem_phy_named_block_find(NULL, flags |
  914. CVMX_BOOTMEM_FLAG_NO_LOCKING);
  915. if (!named_block_desc_addr) {
  916. __cvmx_bootmem_unlock(flags);
  917. return -1;
  918. }
  919. /*
  920. * Round size up to mult of minimum alignment bytes
  921. * We need the actual size allocated to allow for blocks to be
  922. * coallesced when they are freed. The alloc routine does the
  923. * same rounding up on all allocations.
  924. */
  925. size = (size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
  926. ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
  927. addr_allocated = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
  928. alignment,
  929. flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
  930. if (addr_allocated >= 0) {
  931. CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_desc_addr, base_addr,
  932. addr_allocated);
  933. CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_desc_addr, size, size);
  934. CVMX_BOOTMEM_NAMED_SET_NAME(named_block_desc_addr, name,
  935. CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len));
  936. }
  937. __cvmx_bootmem_unlock(flags);
  938. return addr_allocated;
  939. }
  940. void cvmx_bootmem_phy_named_block_print(void)
  941. {
  942. int i;
  943. int printed = 0;
  944. u64 named_block_array_addr =
  945. CVMX_BOOTMEM_DESC_GET_FIELD(named_block_array_addr);
  946. int num_blocks = CVMX_BOOTMEM_DESC_GET_FIELD(named_block_num_blocks);
  947. int name_length = CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len);
  948. u64 named_block_addr = named_block_array_addr;
  949. debug("%s: desc addr: 0x%llx\n",
  950. __func__, CAST_ULL(cvmx_bootmem_desc_addr));
  951. if (__cvmx_bootmem_check_version(3))
  952. return;
  953. printf("List of currently allocated named bootmem blocks:\n");
  954. for (i = 0; i < num_blocks; i++) {
  955. u64 named_size =
  956. CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr, size);
  957. if (named_size) {
  958. char name_tmp[name_length + 1];
  959. u64 named_addr =
  960. CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr,
  961. base_addr);
  962. CVMX_BOOTMEM_NAMED_GET_NAME(named_block_addr, name_tmp,
  963. name_length);
  964. printed++;
  965. printf("Name: %s, address: 0x%08llx, size: 0x%08llx, index: %d\n", name_tmp,
  966. CAST_ULL(named_addr),
  967. CAST_ULL(named_size), i);
  968. }
  969. named_block_addr +=
  970. sizeof(struct cvmx_bootmem_named_block_desc);
  971. }
  972. if (!printed)
  973. printf("No named bootmem blocks exist.\n");
  974. }
  975. s64 cvmx_bootmem_phy_mem_list_init(u64 mem_size,
  976. u32 low_reserved_bytes,
  977. struct cvmx_bootmem_desc *desc_buffer)
  978. {
  979. u64 cur_block_addr;
  980. s64 addr;
  981. int i;
  982. debug("%s (arg desc ptr: %p, cvmx_bootmem_desc: 0x%llx)\n",
  983. __func__, desc_buffer, CAST_ULL(cvmx_bootmem_desc_addr));
  984. /*
  985. * Descriptor buffer needs to be in 32 bit addressable space to be
  986. * compatible with 32 bit applications
  987. */
  988. if (!desc_buffer) {
  989. debug("ERROR: no memory for cvmx_bootmem descriptor provided\n");
  990. return 0;
  991. }
  992. if (mem_size > OCTEON_MAX_PHY_MEM_SIZE) {
  993. mem_size = OCTEON_MAX_PHY_MEM_SIZE;
  994. debug("ERROR: requested memory size too large, truncating to maximum size\n");
  995. }
  996. if (cvmx_bootmem_desc_addr)
  997. return 1;
  998. /* Initialize cvmx pointer to descriptor */
  999. cvmx_bootmem_init(cvmx_ptr_to_phys(desc_buffer));
  1000. /* Fill the bootmem descriptor */
  1001. CVMX_BOOTMEM_DESC_SET_FIELD(lock, 0);
  1002. CVMX_BOOTMEM_DESC_SET_FIELD(flags, 0);
  1003. CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, 0);
  1004. CVMX_BOOTMEM_DESC_SET_FIELD(major_version, CVMX_BOOTMEM_DESC_MAJ_VER);
  1005. CVMX_BOOTMEM_DESC_SET_FIELD(minor_version, CVMX_BOOTMEM_DESC_MIN_VER);
  1006. CVMX_BOOTMEM_DESC_SET_FIELD(app_data_addr, 0);
  1007. CVMX_BOOTMEM_DESC_SET_FIELD(app_data_size, 0);
  1008. /*
  1009. * Set up global pointer to start of list, exclude low 64k for exception
  1010. * vectors, space for global descriptor
  1011. */
  1012. cur_block_addr = (OCTEON_DDR0_BASE + low_reserved_bytes);
  1013. if (mem_size <= OCTEON_DDR0_SIZE) {
  1014. __cvmx_bootmem_phy_free(cur_block_addr,
  1015. mem_size - low_reserved_bytes, 0);
  1016. goto frees_done;
  1017. }
  1018. __cvmx_bootmem_phy_free(cur_block_addr,
  1019. OCTEON_DDR0_SIZE - low_reserved_bytes, 0);
  1020. mem_size -= OCTEON_DDR0_SIZE;
  1021. /* Add DDR2 block next if present */
  1022. if (mem_size > OCTEON_DDR1_SIZE) {
  1023. __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, OCTEON_DDR1_SIZE, 0);
  1024. __cvmx_bootmem_phy_free(OCTEON_DDR2_BASE,
  1025. mem_size - OCTEON_DDR1_SIZE, 0);
  1026. } else {
  1027. __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, mem_size, 0);
  1028. }
  1029. frees_done:
  1030. /* Initialize the named block structure */
  1031. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_name_len, CVMX_BOOTMEM_NAME_LEN);
  1032. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_num_blocks,
  1033. CVMX_BOOTMEM_NUM_NAMED_BLOCKS);
  1034. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, 0);
  1035. /* Allocate this near the top of the low 256 MBytes of memory */
  1036. addr = cvmx_bootmem_phy_alloc(CVMX_BOOTMEM_NUM_NAMED_BLOCKS *
  1037. sizeof(struct cvmx_bootmem_named_block_desc),
  1038. 0, 0x10000000, 0,
  1039. CVMX_BOOTMEM_FLAG_END_ALLOC);
  1040. if (addr >= 0)
  1041. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, addr);
  1042. debug("%s: named_block_array_addr: 0x%llx)\n",
  1043. __func__, CAST_ULL(addr));
  1044. if (addr < 0) {
  1045. debug("FATAL ERROR: unable to allocate memory for bootmem descriptor!\n");
  1046. return 0;
  1047. }
  1048. for (i = 0; i < CVMX_BOOTMEM_NUM_NAMED_BLOCKS; i++) {
  1049. CVMX_BOOTMEM_NAMED_SET_FIELD(addr, base_addr, 0);
  1050. CVMX_BOOTMEM_NAMED_SET_FIELD(addr, size, 0);
  1051. addr += sizeof(struct cvmx_bootmem_named_block_desc);
  1052. }
  1053. return 1;
  1054. }
  1055. s64 cvmx_bootmem_phy_mem_list_init_multi(u8 node_mask,
  1056. u32 mem_sizes[],
  1057. u32 low_reserved_bytes,
  1058. struct cvmx_bootmem_desc *desc_buffer)
  1059. {
  1060. u64 cur_block_addr;
  1061. u64 mem_size;
  1062. s64 addr;
  1063. int i;
  1064. int node;
  1065. u64 node_base; /* Make u64 to reduce type casting */
  1066. mem_sizes[0] = gd->ram_size / (1024 * 1024);
  1067. debug("cvmx_bootmem_phy_mem_list_init (arg desc ptr: %p, cvmx_bootmem_desc: 0x%llx)\n",
  1068. desc_buffer, CAST_ULL(cvmx_bootmem_desc_addr));
  1069. /*
  1070. * Descriptor buffer needs to be in 32 bit addressable space to be
  1071. * compatible with 32 bit applications
  1072. */
  1073. if (!desc_buffer) {
  1074. debug("ERROR: no memory for cvmx_bootmem descriptor provided\n");
  1075. return 0;
  1076. }
  1077. cvmx_coremask_for_each_node(node, node_mask) {
  1078. if ((mem_sizes[node] * 1024 * 1024) > OCTEON_MAX_PHY_MEM_SIZE) {
  1079. mem_sizes[node] = OCTEON_MAX_PHY_MEM_SIZE /
  1080. (1024 * 1024);
  1081. debug("ERROR node#%lld: requested memory size too large, truncating to maximum size\n",
  1082. CAST_ULL(node));
  1083. }
  1084. }
  1085. if (cvmx_bootmem_desc_addr)
  1086. return 1;
  1087. /* Initialize cvmx pointer to descriptor */
  1088. cvmx_bootmem_init(cvmx_ptr_to_phys(desc_buffer));
  1089. /* Fill the bootmem descriptor */
  1090. CVMX_BOOTMEM_DESC_SET_FIELD(lock, 0);
  1091. CVMX_BOOTMEM_DESC_SET_FIELD(flags, 0);
  1092. CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, 0);
  1093. CVMX_BOOTMEM_DESC_SET_FIELD(major_version, CVMX_BOOTMEM_DESC_MAJ_VER);
  1094. CVMX_BOOTMEM_DESC_SET_FIELD(minor_version, CVMX_BOOTMEM_DESC_MIN_VER);
  1095. CVMX_BOOTMEM_DESC_SET_FIELD(app_data_addr, 0);
  1096. CVMX_BOOTMEM_DESC_SET_FIELD(app_data_size, 0);
  1097. cvmx_coremask_for_each_node(node, node_mask) {
  1098. if (node != 0) /* do not reserve memory on remote nodes */
  1099. low_reserved_bytes = 0;
  1100. mem_size = (u64)mem_sizes[node] * (1024 * 1024); /* MBytes */
  1101. /*
  1102. * Set up global pointer to start of list, exclude low 64k
  1103. * for exception vectors, space for global descriptor
  1104. */
  1105. node_base = (u64)node << CVMX_NODE_MEM_SHIFT;
  1106. cur_block_addr = (OCTEON_DDR0_BASE + low_reserved_bytes) |
  1107. node_base;
  1108. if (mem_size <= OCTEON_DDR0_SIZE) {
  1109. __cvmx_bootmem_phy_free(cur_block_addr,
  1110. mem_size - low_reserved_bytes,
  1111. 0);
  1112. continue;
  1113. }
  1114. __cvmx_bootmem_phy_free(cur_block_addr,
  1115. OCTEON_DDR0_SIZE - low_reserved_bytes,
  1116. 0);
  1117. mem_size -= OCTEON_DDR0_SIZE;
  1118. /* Add DDR2 block next if present */
  1119. if (mem_size > OCTEON_DDR1_SIZE) {
  1120. __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE |
  1121. node_base,
  1122. OCTEON_DDR1_SIZE, 0);
  1123. __cvmx_bootmem_phy_free(OCTEON_DDR2_BASE |
  1124. node_base,
  1125. mem_size - OCTEON_DDR1_SIZE, 0);
  1126. } else {
  1127. __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE |
  1128. node_base,
  1129. mem_size, 0);
  1130. }
  1131. }
  1132. debug("%s: Initialize the named block\n", __func__);
  1133. /* Initialize the named block structure */
  1134. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_name_len, CVMX_BOOTMEM_NAME_LEN);
  1135. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_num_blocks,
  1136. CVMX_BOOTMEM_NUM_NAMED_BLOCKS);
  1137. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, 0);
  1138. /* Allocate this near the top of the low 256 MBytes of memory */
  1139. addr = cvmx_bootmem_phy_alloc(CVMX_BOOTMEM_NUM_NAMED_BLOCKS *
  1140. sizeof(struct cvmx_bootmem_named_block_desc),
  1141. 0, 0x10000000, 0,
  1142. CVMX_BOOTMEM_FLAG_END_ALLOC);
  1143. if (addr >= 0)
  1144. CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, addr);
  1145. debug("cvmx_bootmem_phy_mem_list_init: named_block_array_addr: 0x%llx)\n",
  1146. CAST_ULL(addr));
  1147. if (addr < 0) {
  1148. debug("FATAL ERROR: unable to allocate memory for bootmem descriptor!\n");
  1149. return 0;
  1150. }
  1151. for (i = 0; i < CVMX_BOOTMEM_NUM_NAMED_BLOCKS; i++) {
  1152. CVMX_BOOTMEM_NAMED_SET_FIELD(addr, base_addr, 0);
  1153. CVMX_BOOTMEM_NAMED_SET_FIELD(addr, size, 0);
  1154. addr += sizeof(struct cvmx_bootmem_named_block_desc);
  1155. }
  1156. // test-only: DEBUG ifdef???
  1157. cvmx_bootmem_phy_list_print();
  1158. return 1;
  1159. }
  1160. int cvmx_bootmem_reserve_memory(u64 start_addr, u64 size,
  1161. const char *name, u32 flags)
  1162. {
  1163. u64 addr;
  1164. int rc = 1;
  1165. static unsigned int block_num;
  1166. char block_name[CVMX_BOOTMEM_NAME_LEN];
  1167. debug("%s: start %#llx, size: %#llx, name: %s, flags:%#x)\n",
  1168. __func__, CAST_ULL(start_addr), CAST_ULL(size), name, flags);
  1169. if (__cvmx_bootmem_check_version(3))
  1170. return 0;
  1171. addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
  1172. if (!addr)
  1173. return 0;
  1174. if (!name)
  1175. name = "__cvmx_bootmem_reserved";
  1176. while (addr && rc) {
  1177. u64 block_size = cvmx_bootmem_phy_get_size(addr);
  1178. u64 reserve_size = 0;
  1179. if (addr >= start_addr && addr < start_addr + size) {
  1180. reserve_size = size - (addr - start_addr);
  1181. if (block_size < reserve_size)
  1182. reserve_size = block_size;
  1183. } else if (start_addr > addr &&
  1184. start_addr < (addr + block_size)) {
  1185. reserve_size = block_size - (start_addr - addr);
  1186. }
  1187. if (reserve_size) {
  1188. snprintf(block_name, sizeof(block_name),
  1189. "%.32s_%012llx_%u",
  1190. name, (unsigned long long)start_addr,
  1191. (unsigned int)block_num);
  1192. debug("%s: Reserving 0x%llx bytes at address 0x%llx with name %s\n",
  1193. __func__, CAST_ULL(reserve_size),
  1194. CAST_ULL(addr), block_name);
  1195. if (cvmx_bootmem_phy_named_block_alloc(reserve_size,
  1196. addr, 0, 0,
  1197. block_name,
  1198. flags) == -1) {
  1199. debug("%s: Failed to reserve 0x%llx bytes at address 0x%llx\n",
  1200. __func__, CAST_ULL(reserve_size),
  1201. (unsigned long long)addr);
  1202. rc = 0;
  1203. break;
  1204. }
  1205. debug("%s: Reserved 0x%llx bytes at address 0x%llx with name %s\n",
  1206. __func__, CAST_ULL(reserve_size),
  1207. CAST_ULL(addr), block_name);
  1208. }
  1209. addr = cvmx_bootmem_phy_get_next(addr);
  1210. block_num++;
  1211. }
  1212. return rc;
  1213. }
  1214. void cvmx_bootmem_lock(void)
  1215. {
  1216. __cvmx_bootmem_lock(0);
  1217. }
  1218. void cvmx_bootmem_unlock(void)
  1219. {
  1220. __cvmx_bootmem_unlock(0);
  1221. }
  1222. void *__cvmx_phys_addr_to_ptr(u64 phys, int size)
  1223. {
  1224. void *tmp;
  1225. if (sizeof(void *) == 8) {
  1226. tmp = CASTPTR(void, CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, phys));
  1227. } else {
  1228. u32 phy32 = (u32)(phys & 0x7fffffffULL);
  1229. tmp = CASTPTR(void, CVMX_ADD_SEG32(CVMX_MIPS32_SPACE_KSEG0,
  1230. phy32));
  1231. }
  1232. return tmp;
  1233. }
  1234. void *__cvmx_bootmem_internal_get_desc_ptr(void)
  1235. {
  1236. return cvmx_phys_to_ptr(cvmx_bootmem_desc_addr);
  1237. }