wl.c 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Copyright (c) International Business Machines Corp., 2006
  4. *
  5. * Authors: Artem Bityutskiy (Битюцкий Артём), Thomas Gleixner
  6. */
  7. /*
  8. * UBI wear-leveling sub-system.
  9. *
  10. * This sub-system is responsible for wear-leveling. It works in terms of
  11. * physical eraseblocks and erase counters and knows nothing about logical
  12. * eraseblocks, volumes, etc. From this sub-system's perspective all physical
  13. * eraseblocks are of two types - used and free. Used physical eraseblocks are
  14. * those that were "get" by the 'ubi_wl_get_peb()' function, and free physical
  15. * eraseblocks are those that were put by the 'ubi_wl_put_peb()' function.
  16. *
  17. * Physical eraseblocks returned by 'ubi_wl_get_peb()' have only erase counter
  18. * header. The rest of the physical eraseblock contains only %0xFF bytes.
  19. *
  20. * When physical eraseblocks are returned to the WL sub-system by means of the
  21. * 'ubi_wl_put_peb()' function, they are scheduled for erasure. The erasure is
  22. * done asynchronously in context of the per-UBI device background thread,
  23. * which is also managed by the WL sub-system.
  24. *
  25. * The wear-leveling is ensured by means of moving the contents of used
  26. * physical eraseblocks with low erase counter to free physical eraseblocks
  27. * with high erase counter.
  28. *
  29. * If the WL sub-system fails to erase a physical eraseblock, it marks it as
  30. * bad.
  31. *
  32. * This sub-system is also responsible for scrubbing. If a bit-flip is detected
  33. * in a physical eraseblock, it has to be moved. Technically this is the same
  34. * as moving it for wear-leveling reasons.
  35. *
  36. * As it was said, for the UBI sub-system all physical eraseblocks are either
  37. * "free" or "used". Free eraseblock are kept in the @wl->free RB-tree, while
  38. * used eraseblocks are kept in @wl->used, @wl->erroneous, or @wl->scrub
  39. * RB-trees, as well as (temporarily) in the @wl->pq queue.
  40. *
  41. * When the WL sub-system returns a physical eraseblock, the physical
  42. * eraseblock is protected from being moved for some "time". For this reason,
  43. * the physical eraseblock is not directly moved from the @wl->free tree to the
  44. * @wl->used tree. There is a protection queue in between where this
  45. * physical eraseblock is temporarily stored (@wl->pq).
  46. *
  47. * All this protection stuff is needed because:
  48. * o we don't want to move physical eraseblocks just after we have given them
  49. * to the user; instead, we first want to let users fill them up with data;
  50. *
  51. * o there is a chance that the user will put the physical eraseblock very
  52. * soon, so it makes sense not to move it for some time, but wait.
  53. *
  54. * Physical eraseblocks stay protected only for limited time. But the "time" is
  55. * measured in erase cycles in this case. This is implemented with help of the
  56. * protection queue. Eraseblocks are put to the tail of this queue when they
  57. * are returned by the 'ubi_wl_get_peb()', and eraseblocks are removed from the
  58. * head of the queue on each erase operation (for any eraseblock). So the
  59. * length of the queue defines how may (global) erase cycles PEBs are protected.
  60. *
  61. * To put it differently, each physical eraseblock has 2 main states: free and
  62. * used. The former state corresponds to the @wl->free tree. The latter state
  63. * is split up on several sub-states:
  64. * o the WL movement is allowed (@wl->used tree);
  65. * o the WL movement is disallowed (@wl->erroneous) because the PEB is
  66. * erroneous - e.g., there was a read error;
  67. * o the WL movement is temporarily prohibited (@wl->pq queue);
  68. * o scrubbing is needed (@wl->scrub tree).
  69. *
  70. * Depending on the sub-state, wear-leveling entries of the used physical
  71. * eraseblocks may be kept in one of those structures.
  72. *
  73. * Note, in this implementation, we keep a small in-RAM object for each physical
  74. * eraseblock. This is surely not a scalable solution. But it appears to be good
  75. * enough for moderately large flashes and it is simple. In future, one may
  76. * re-work this sub-system and make it more scalable.
  77. *
  78. * At the moment this sub-system does not utilize the sequence number, which
  79. * was introduced relatively recently. But it would be wise to do this because
  80. * the sequence number of a logical eraseblock characterizes how old is it. For
  81. * example, when we move a PEB with low erase counter, and we need to pick the
  82. * target PEB, we pick a PEB with the highest EC if our PEB is "old" and we
  83. * pick target PEB with an average EC if our PEB is not very "old". This is a
  84. * room for future re-works of the WL sub-system.
  85. */
  86. #ifndef __UBOOT__
  87. #include <linux/slab.h>
  88. #include <linux/crc32.h>
  89. #include <linux/freezer.h>
  90. #include <linux/kthread.h>
  91. #else
  92. #include <ubi_uboot.h>
  93. #endif
  94. #include "ubi.h"
  95. #include "wl.h"
  96. /* Number of physical eraseblocks reserved for wear-leveling purposes */
  97. #define WL_RESERVED_PEBS 1
  98. /*
  99. * Maximum difference between two erase counters. If this threshold is
  100. * exceeded, the WL sub-system starts moving data from used physical
  101. * eraseblocks with low erase counter to free physical eraseblocks with high
  102. * erase counter.
  103. */
  104. #define UBI_WL_THRESHOLD CONFIG_MTD_UBI_WL_THRESHOLD
  105. /*
  106. * When a physical eraseblock is moved, the WL sub-system has to pick the target
  107. * physical eraseblock to move to. The simplest way would be just to pick the
  108. * one with the highest erase counter. But in certain workloads this could lead
  109. * to an unlimited wear of one or few physical eraseblock. Indeed, imagine a
  110. * situation when the picked physical eraseblock is constantly erased after the
  111. * data is written to it. So, we have a constant which limits the highest erase
  112. * counter of the free physical eraseblock to pick. Namely, the WL sub-system
  113. * does not pick eraseblocks with erase counter greater than the lowest erase
  114. * counter plus %WL_FREE_MAX_DIFF.
  115. */
  116. #define WL_FREE_MAX_DIFF (2*UBI_WL_THRESHOLD)
  117. /*
  118. * Maximum number of consecutive background thread failures which is enough to
  119. * switch to read-only mode.
  120. */
  121. #define WL_MAX_FAILURES 32
  122. static int self_check_ec(struct ubi_device *ubi, int pnum, int ec);
  123. static int self_check_in_wl_tree(const struct ubi_device *ubi,
  124. struct ubi_wl_entry *e, struct rb_root *root);
  125. static int self_check_in_pq(const struct ubi_device *ubi,
  126. struct ubi_wl_entry *e);
  127. /**
  128. * wl_tree_add - add a wear-leveling entry to a WL RB-tree.
  129. * @e: the wear-leveling entry to add
  130. * @root: the root of the tree
  131. *
  132. * Note, we use (erase counter, physical eraseblock number) pairs as keys in
  133. * the @ubi->used and @ubi->free RB-trees.
  134. */
  135. static void wl_tree_add(struct ubi_wl_entry *e, struct rb_root *root)
  136. {
  137. struct rb_node **p, *parent = NULL;
  138. p = &root->rb_node;
  139. while (*p) {
  140. struct ubi_wl_entry *e1;
  141. parent = *p;
  142. e1 = rb_entry(parent, struct ubi_wl_entry, u.rb);
  143. if (e->ec < e1->ec)
  144. p = &(*p)->rb_left;
  145. else if (e->ec > e1->ec)
  146. p = &(*p)->rb_right;
  147. else {
  148. ubi_assert(e->pnum != e1->pnum);
  149. if (e->pnum < e1->pnum)
  150. p = &(*p)->rb_left;
  151. else
  152. p = &(*p)->rb_right;
  153. }
  154. }
  155. rb_link_node(&e->u.rb, parent, p);
  156. rb_insert_color(&e->u.rb, root);
  157. }
  158. /**
  159. * wl_tree_destroy - destroy a wear-leveling entry.
  160. * @ubi: UBI device description object
  161. * @e: the wear-leveling entry to add
  162. *
  163. * This function destroys a wear leveling entry and removes
  164. * the reference from the lookup table.
  165. */
  166. static void wl_entry_destroy(struct ubi_device *ubi, struct ubi_wl_entry *e)
  167. {
  168. ubi->lookuptbl[e->pnum] = NULL;
  169. kmem_cache_free(ubi_wl_entry_slab, e);
  170. }
  171. /**
  172. * do_work - do one pending work.
  173. * @ubi: UBI device description object
  174. *
  175. * This function returns zero in case of success and a negative error code in
  176. * case of failure.
  177. */
  178. static int do_work(struct ubi_device *ubi)
  179. {
  180. int err;
  181. struct ubi_work *wrk;
  182. cond_resched();
  183. /*
  184. * @ubi->work_sem is used to synchronize with the workers. Workers take
  185. * it in read mode, so many of them may be doing works at a time. But
  186. * the queue flush code has to be sure the whole queue of works is
  187. * done, and it takes the mutex in write mode.
  188. */
  189. down_read(&ubi->work_sem);
  190. spin_lock(&ubi->wl_lock);
  191. if (list_empty(&ubi->works)) {
  192. spin_unlock(&ubi->wl_lock);
  193. up_read(&ubi->work_sem);
  194. return 0;
  195. }
  196. wrk = list_entry(ubi->works.next, struct ubi_work, list);
  197. list_del(&wrk->list);
  198. ubi->works_count -= 1;
  199. ubi_assert(ubi->works_count >= 0);
  200. spin_unlock(&ubi->wl_lock);
  201. /*
  202. * Call the worker function. Do not touch the work structure
  203. * after this call as it will have been freed or reused by that
  204. * time by the worker function.
  205. */
  206. err = wrk->func(ubi, wrk, 0);
  207. if (err)
  208. ubi_err(ubi, "work failed with error code %d", err);
  209. up_read(&ubi->work_sem);
  210. return err;
  211. }
  212. /**
  213. * in_wl_tree - check if wear-leveling entry is present in a WL RB-tree.
  214. * @e: the wear-leveling entry to check
  215. * @root: the root of the tree
  216. *
  217. * This function returns non-zero if @e is in the @root RB-tree and zero if it
  218. * is not.
  219. */
  220. static int in_wl_tree(struct ubi_wl_entry *e, struct rb_root *root)
  221. {
  222. struct rb_node *p;
  223. p = root->rb_node;
  224. while (p) {
  225. struct ubi_wl_entry *e1;
  226. e1 = rb_entry(p, struct ubi_wl_entry, u.rb);
  227. if (e->pnum == e1->pnum) {
  228. ubi_assert(e == e1);
  229. return 1;
  230. }
  231. if (e->ec < e1->ec)
  232. p = p->rb_left;
  233. else if (e->ec > e1->ec)
  234. p = p->rb_right;
  235. else {
  236. ubi_assert(e->pnum != e1->pnum);
  237. if (e->pnum < e1->pnum)
  238. p = p->rb_left;
  239. else
  240. p = p->rb_right;
  241. }
  242. }
  243. return 0;
  244. }
  245. /**
  246. * prot_queue_add - add physical eraseblock to the protection queue.
  247. * @ubi: UBI device description object
  248. * @e: the physical eraseblock to add
  249. *
  250. * This function adds @e to the tail of the protection queue @ubi->pq, where
  251. * @e will stay for %UBI_PROT_QUEUE_LEN erase operations and will be
  252. * temporarily protected from the wear-leveling worker. Note, @wl->lock has to
  253. * be locked.
  254. */
  255. static void prot_queue_add(struct ubi_device *ubi, struct ubi_wl_entry *e)
  256. {
  257. int pq_tail = ubi->pq_head - 1;
  258. if (pq_tail < 0)
  259. pq_tail = UBI_PROT_QUEUE_LEN - 1;
  260. ubi_assert(pq_tail >= 0 && pq_tail < UBI_PROT_QUEUE_LEN);
  261. list_add_tail(&e->u.list, &ubi->pq[pq_tail]);
  262. dbg_wl("added PEB %d EC %d to the protection queue", e->pnum, e->ec);
  263. }
  264. /**
  265. * find_wl_entry - find wear-leveling entry closest to certain erase counter.
  266. * @ubi: UBI device description object
  267. * @root: the RB-tree where to look for
  268. * @diff: maximum possible difference from the smallest erase counter
  269. *
  270. * This function looks for a wear leveling entry with erase counter closest to
  271. * min + @diff, where min is the smallest erase counter.
  272. */
  273. static struct ubi_wl_entry *find_wl_entry(struct ubi_device *ubi,
  274. struct rb_root *root, int diff)
  275. {
  276. struct rb_node *p;
  277. struct ubi_wl_entry *e, *prev_e = NULL;
  278. int max;
  279. e = rb_entry(rb_first(root), struct ubi_wl_entry, u.rb);
  280. max = e->ec + diff;
  281. p = root->rb_node;
  282. while (p) {
  283. struct ubi_wl_entry *e1;
  284. e1 = rb_entry(p, struct ubi_wl_entry, u.rb);
  285. if (e1->ec >= max)
  286. p = p->rb_left;
  287. else {
  288. p = p->rb_right;
  289. prev_e = e;
  290. e = e1;
  291. }
  292. }
  293. /* If no fastmap has been written and this WL entry can be used
  294. * as anchor PEB, hold it back and return the second best WL entry
  295. * such that fastmap can use the anchor PEB later. */
  296. if (prev_e && !ubi->fm_disabled &&
  297. !ubi->fm && e->pnum < UBI_FM_MAX_START)
  298. return prev_e;
  299. return e;
  300. }
  301. /**
  302. * find_mean_wl_entry - find wear-leveling entry with medium erase counter.
  303. * @ubi: UBI device description object
  304. * @root: the RB-tree where to look for
  305. *
  306. * This function looks for a wear leveling entry with medium erase counter,
  307. * but not greater or equivalent than the lowest erase counter plus
  308. * %WL_FREE_MAX_DIFF/2.
  309. */
  310. static struct ubi_wl_entry *find_mean_wl_entry(struct ubi_device *ubi,
  311. struct rb_root *root)
  312. {
  313. struct ubi_wl_entry *e, *first, *last;
  314. first = rb_entry(rb_first(root), struct ubi_wl_entry, u.rb);
  315. last = rb_entry(rb_last(root), struct ubi_wl_entry, u.rb);
  316. if (last->ec - first->ec < WL_FREE_MAX_DIFF) {
  317. e = rb_entry(root->rb_node, struct ubi_wl_entry, u.rb);
  318. /* If no fastmap has been written and this WL entry can be used
  319. * as anchor PEB, hold it back and return the second best
  320. * WL entry such that fastmap can use the anchor PEB later. */
  321. e = may_reserve_for_fm(ubi, e, root);
  322. } else
  323. e = find_wl_entry(ubi, root, WL_FREE_MAX_DIFF/2);
  324. return e;
  325. }
  326. /**
  327. * wl_get_wle - get a mean wl entry to be used by ubi_wl_get_peb() or
  328. * refill_wl_user_pool().
  329. * @ubi: UBI device description object
  330. *
  331. * This function returns a a wear leveling entry in case of success and
  332. * NULL in case of failure.
  333. */
  334. static struct ubi_wl_entry *wl_get_wle(struct ubi_device *ubi)
  335. {
  336. struct ubi_wl_entry *e;
  337. e = find_mean_wl_entry(ubi, &ubi->free);
  338. if (!e) {
  339. ubi_err(ubi, "no free eraseblocks");
  340. return NULL;
  341. }
  342. self_check_in_wl_tree(ubi, e, &ubi->free);
  343. /*
  344. * Move the physical eraseblock to the protection queue where it will
  345. * be protected from being moved for some time.
  346. */
  347. rb_erase(&e->u.rb, &ubi->free);
  348. ubi->free_count--;
  349. dbg_wl("PEB %d EC %d", e->pnum, e->ec);
  350. return e;
  351. }
  352. /**
  353. * prot_queue_del - remove a physical eraseblock from the protection queue.
  354. * @ubi: UBI device description object
  355. * @pnum: the physical eraseblock to remove
  356. *
  357. * This function deletes PEB @pnum from the protection queue and returns zero
  358. * in case of success and %-ENODEV if the PEB was not found.
  359. */
  360. static int prot_queue_del(struct ubi_device *ubi, int pnum)
  361. {
  362. struct ubi_wl_entry *e;
  363. e = ubi->lookuptbl[pnum];
  364. if (!e)
  365. return -ENODEV;
  366. if (self_check_in_pq(ubi, e))
  367. return -ENODEV;
  368. list_del(&e->u.list);
  369. dbg_wl("deleted PEB %d from the protection queue", e->pnum);
  370. return 0;
  371. }
  372. /**
  373. * sync_erase - synchronously erase a physical eraseblock.
  374. * @ubi: UBI device description object
  375. * @e: the the physical eraseblock to erase
  376. * @torture: if the physical eraseblock has to be tortured
  377. *
  378. * This function returns zero in case of success and a negative error code in
  379. * case of failure.
  380. */
  381. static int sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e,
  382. int torture)
  383. {
  384. int err;
  385. struct ubi_ec_hdr *ec_hdr;
  386. unsigned long long ec = e->ec;
  387. dbg_wl("erase PEB %d, old EC %llu", e->pnum, ec);
  388. err = self_check_ec(ubi, e->pnum, e->ec);
  389. if (err)
  390. return -EINVAL;
  391. ec_hdr = kzalloc(ubi->ec_hdr_alsize, GFP_NOFS);
  392. if (!ec_hdr)
  393. return -ENOMEM;
  394. err = ubi_io_sync_erase(ubi, e->pnum, torture);
  395. if (err < 0)
  396. goto out_free;
  397. ec += err;
  398. if (ec > UBI_MAX_ERASECOUNTER) {
  399. /*
  400. * Erase counter overflow. Upgrade UBI and use 64-bit
  401. * erase counters internally.
  402. */
  403. ubi_err(ubi, "erase counter overflow at PEB %d, EC %llu",
  404. e->pnum, ec);
  405. err = -EINVAL;
  406. goto out_free;
  407. }
  408. dbg_wl("erased PEB %d, new EC %llu", e->pnum, ec);
  409. ec_hdr->ec = cpu_to_be64(ec);
  410. err = ubi_io_write_ec_hdr(ubi, e->pnum, ec_hdr);
  411. if (err)
  412. goto out_free;
  413. e->ec = ec;
  414. spin_lock(&ubi->wl_lock);
  415. if (e->ec > ubi->max_ec)
  416. ubi->max_ec = e->ec;
  417. spin_unlock(&ubi->wl_lock);
  418. out_free:
  419. kfree(ec_hdr);
  420. return err;
  421. }
  422. /**
  423. * serve_prot_queue - check if it is time to stop protecting PEBs.
  424. * @ubi: UBI device description object
  425. *
  426. * This function is called after each erase operation and removes PEBs from the
  427. * tail of the protection queue. These PEBs have been protected for long enough
  428. * and should be moved to the used tree.
  429. */
  430. static void serve_prot_queue(struct ubi_device *ubi)
  431. {
  432. struct ubi_wl_entry *e, *tmp;
  433. int count;
  434. /*
  435. * There may be several protected physical eraseblock to remove,
  436. * process them all.
  437. */
  438. repeat:
  439. count = 0;
  440. spin_lock(&ubi->wl_lock);
  441. list_for_each_entry_safe(e, tmp, &ubi->pq[ubi->pq_head], u.list) {
  442. dbg_wl("PEB %d EC %d protection over, move to used tree",
  443. e->pnum, e->ec);
  444. list_del(&e->u.list);
  445. wl_tree_add(e, &ubi->used);
  446. if (count++ > 32) {
  447. /*
  448. * Let's be nice and avoid holding the spinlock for
  449. * too long.
  450. */
  451. spin_unlock(&ubi->wl_lock);
  452. cond_resched();
  453. goto repeat;
  454. }
  455. }
  456. ubi->pq_head += 1;
  457. if (ubi->pq_head == UBI_PROT_QUEUE_LEN)
  458. ubi->pq_head = 0;
  459. ubi_assert(ubi->pq_head >= 0 && ubi->pq_head < UBI_PROT_QUEUE_LEN);
  460. spin_unlock(&ubi->wl_lock);
  461. }
  462. #ifdef __UBOOT__
  463. void ubi_do_worker(struct ubi_device *ubi)
  464. {
  465. int err;
  466. if (list_empty(&ubi->works) || ubi->ro_mode ||
  467. !ubi->thread_enabled || ubi_dbg_is_bgt_disabled(ubi))
  468. return;
  469. spin_lock(&ubi->wl_lock);
  470. while (!list_empty(&ubi->works)) {
  471. /*
  472. * call do_work, which executes exactly one work form the queue,
  473. * including removeing it from the work queue.
  474. */
  475. spin_unlock(&ubi->wl_lock);
  476. err = do_work(ubi);
  477. spin_lock(&ubi->wl_lock);
  478. if (err) {
  479. ubi_err(ubi, "%s: work failed with error code %d",
  480. ubi->bgt_name, err);
  481. }
  482. }
  483. spin_unlock(&ubi->wl_lock);
  484. }
  485. #endif
  486. /**
  487. * __schedule_ubi_work - schedule a work.
  488. * @ubi: UBI device description object
  489. * @wrk: the work to schedule
  490. *
  491. * This function adds a work defined by @wrk to the tail of the pending works
  492. * list. Can only be used if ubi->work_sem is already held in read mode!
  493. */
  494. static void __schedule_ubi_work(struct ubi_device *ubi, struct ubi_work *wrk)
  495. {
  496. spin_lock(&ubi->wl_lock);
  497. list_add_tail(&wrk->list, &ubi->works);
  498. ubi_assert(ubi->works_count >= 0);
  499. ubi->works_count += 1;
  500. #ifndef __UBOOT__
  501. if (ubi->thread_enabled && !ubi_dbg_is_bgt_disabled(ubi))
  502. wake_up_process(ubi->bgt_thread);
  503. #endif
  504. spin_unlock(&ubi->wl_lock);
  505. }
  506. /**
  507. * schedule_ubi_work - schedule a work.
  508. * @ubi: UBI device description object
  509. * @wrk: the work to schedule
  510. *
  511. * This function adds a work defined by @wrk to the tail of the pending works
  512. * list.
  513. */
  514. static void schedule_ubi_work(struct ubi_device *ubi, struct ubi_work *wrk)
  515. {
  516. down_read(&ubi->work_sem);
  517. __schedule_ubi_work(ubi, wrk);
  518. up_read(&ubi->work_sem);
  519. }
  520. static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk,
  521. int shutdown);
  522. /**
  523. * schedule_erase - schedule an erase work.
  524. * @ubi: UBI device description object
  525. * @e: the WL entry of the physical eraseblock to erase
  526. * @vol_id: the volume ID that last used this PEB
  527. * @lnum: the last used logical eraseblock number for the PEB
  528. * @torture: if the physical eraseblock has to be tortured
  529. *
  530. * This function returns zero in case of success and a %-ENOMEM in case of
  531. * failure.
  532. */
  533. static int schedule_erase(struct ubi_device *ubi, struct ubi_wl_entry *e,
  534. int vol_id, int lnum, int torture)
  535. {
  536. struct ubi_work *wl_wrk;
  537. ubi_assert(e);
  538. dbg_wl("schedule erasure of PEB %d, EC %d, torture %d",
  539. e->pnum, e->ec, torture);
  540. wl_wrk = kmalloc(sizeof(struct ubi_work), GFP_NOFS);
  541. if (!wl_wrk)
  542. return -ENOMEM;
  543. wl_wrk->func = &erase_worker;
  544. wl_wrk->e = e;
  545. wl_wrk->vol_id = vol_id;
  546. wl_wrk->lnum = lnum;
  547. wl_wrk->torture = torture;
  548. schedule_ubi_work(ubi, wl_wrk);
  549. #ifdef __UBOOT__
  550. ubi_do_worker(ubi);
  551. #endif
  552. return 0;
  553. }
  554. /**
  555. * do_sync_erase - run the erase worker synchronously.
  556. * @ubi: UBI device description object
  557. * @e: the WL entry of the physical eraseblock to erase
  558. * @vol_id: the volume ID that last used this PEB
  559. * @lnum: the last used logical eraseblock number for the PEB
  560. * @torture: if the physical eraseblock has to be tortured
  561. *
  562. */
  563. static int do_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e,
  564. int vol_id, int lnum, int torture)
  565. {
  566. struct ubi_work *wl_wrk;
  567. dbg_wl("sync erase of PEB %i", e->pnum);
  568. wl_wrk = kmalloc(sizeof(struct ubi_work), GFP_NOFS);
  569. if (!wl_wrk)
  570. return -ENOMEM;
  571. wl_wrk->e = e;
  572. wl_wrk->vol_id = vol_id;
  573. wl_wrk->lnum = lnum;
  574. wl_wrk->torture = torture;
  575. return erase_worker(ubi, wl_wrk, 0);
  576. }
  577. /**
  578. * wear_leveling_worker - wear-leveling worker function.
  579. * @ubi: UBI device description object
  580. * @wrk: the work object
  581. * @shutdown: non-zero if the worker has to free memory and exit
  582. * because the WL-subsystem is shutting down
  583. *
  584. * This function copies a more worn out physical eraseblock to a less worn out
  585. * one. Returns zero in case of success and a negative error code in case of
  586. * failure.
  587. */
  588. static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
  589. int shutdown)
  590. {
  591. int err, scrubbing = 0, torture = 0, protect = 0, erroneous = 0;
  592. int vol_id = -1, lnum = -1;
  593. #ifdef CONFIG_MTD_UBI_FASTMAP
  594. int anchor = wrk->anchor;
  595. #endif
  596. struct ubi_wl_entry *e1, *e2;
  597. struct ubi_vid_hdr *vid_hdr;
  598. kfree(wrk);
  599. if (shutdown)
  600. return 0;
  601. vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
  602. if (!vid_hdr)
  603. return -ENOMEM;
  604. mutex_lock(&ubi->move_mutex);
  605. spin_lock(&ubi->wl_lock);
  606. ubi_assert(!ubi->move_from && !ubi->move_to);
  607. ubi_assert(!ubi->move_to_put);
  608. if (!ubi->free.rb_node ||
  609. (!ubi->used.rb_node && !ubi->scrub.rb_node)) {
  610. /*
  611. * No free physical eraseblocks? Well, they must be waiting in
  612. * the queue to be erased. Cancel movement - it will be
  613. * triggered again when a free physical eraseblock appears.
  614. *
  615. * No used physical eraseblocks? They must be temporarily
  616. * protected from being moved. They will be moved to the
  617. * @ubi->used tree later and the wear-leveling will be
  618. * triggered again.
  619. */
  620. dbg_wl("cancel WL, a list is empty: free %d, used %d",
  621. !ubi->free.rb_node, !ubi->used.rb_node);
  622. goto out_cancel;
  623. }
  624. #ifdef CONFIG_MTD_UBI_FASTMAP
  625. /* Check whether we need to produce an anchor PEB */
  626. if (!anchor)
  627. anchor = !anchor_pebs_avalible(&ubi->free);
  628. if (anchor) {
  629. e1 = find_anchor_wl_entry(&ubi->used);
  630. if (!e1)
  631. goto out_cancel;
  632. e2 = get_peb_for_wl(ubi);
  633. if (!e2)
  634. goto out_cancel;
  635. self_check_in_wl_tree(ubi, e1, &ubi->used);
  636. rb_erase(&e1->u.rb, &ubi->used);
  637. dbg_wl("anchor-move PEB %d to PEB %d", e1->pnum, e2->pnum);
  638. } else if (!ubi->scrub.rb_node) {
  639. #else
  640. if (!ubi->scrub.rb_node) {
  641. #endif
  642. /*
  643. * Now pick the least worn-out used physical eraseblock and a
  644. * highly worn-out free physical eraseblock. If the erase
  645. * counters differ much enough, start wear-leveling.
  646. */
  647. e1 = rb_entry(rb_first(&ubi->used), struct ubi_wl_entry, u.rb);
  648. e2 = get_peb_for_wl(ubi);
  649. if (!e2)
  650. goto out_cancel;
  651. if (!(e2->ec - e1->ec >= UBI_WL_THRESHOLD)) {
  652. dbg_wl("no WL needed: min used EC %d, max free EC %d",
  653. e1->ec, e2->ec);
  654. /* Give the unused PEB back */
  655. wl_tree_add(e2, &ubi->free);
  656. ubi->free_count++;
  657. goto out_cancel;
  658. }
  659. self_check_in_wl_tree(ubi, e1, &ubi->used);
  660. rb_erase(&e1->u.rb, &ubi->used);
  661. dbg_wl("move PEB %d EC %d to PEB %d EC %d",
  662. e1->pnum, e1->ec, e2->pnum, e2->ec);
  663. } else {
  664. /* Perform scrubbing */
  665. scrubbing = 1;
  666. e1 = rb_entry(rb_first(&ubi->scrub), struct ubi_wl_entry, u.rb);
  667. e2 = get_peb_for_wl(ubi);
  668. if (!e2)
  669. goto out_cancel;
  670. self_check_in_wl_tree(ubi, e1, &ubi->scrub);
  671. rb_erase(&e1->u.rb, &ubi->scrub);
  672. dbg_wl("scrub PEB %d to PEB %d", e1->pnum, e2->pnum);
  673. }
  674. ubi->move_from = e1;
  675. ubi->move_to = e2;
  676. spin_unlock(&ubi->wl_lock);
  677. /*
  678. * Now we are going to copy physical eraseblock @e1->pnum to @e2->pnum.
  679. * We so far do not know which logical eraseblock our physical
  680. * eraseblock (@e1) belongs to. We have to read the volume identifier
  681. * header first.
  682. *
  683. * Note, we are protected from this PEB being unmapped and erased. The
  684. * 'ubi_wl_put_peb()' would wait for moving to be finished if the PEB
  685. * which is being moved was unmapped.
  686. */
  687. err = ubi_io_read_vid_hdr(ubi, e1->pnum, vid_hdr, 0);
  688. if (err && err != UBI_IO_BITFLIPS) {
  689. if (err == UBI_IO_FF) {
  690. /*
  691. * We are trying to move PEB without a VID header. UBI
  692. * always write VID headers shortly after the PEB was
  693. * given, so we have a situation when it has not yet
  694. * had a chance to write it, because it was preempted.
  695. * So add this PEB to the protection queue so far,
  696. * because presumably more data will be written there
  697. * (including the missing VID header), and then we'll
  698. * move it.
  699. */
  700. dbg_wl("PEB %d has no VID header", e1->pnum);
  701. protect = 1;
  702. goto out_not_moved;
  703. } else if (err == UBI_IO_FF_BITFLIPS) {
  704. /*
  705. * The same situation as %UBI_IO_FF, but bit-flips were
  706. * detected. It is better to schedule this PEB for
  707. * scrubbing.
  708. */
  709. dbg_wl("PEB %d has no VID header but has bit-flips",
  710. e1->pnum);
  711. scrubbing = 1;
  712. goto out_not_moved;
  713. }
  714. ubi_err(ubi, "error %d while reading VID header from PEB %d",
  715. err, e1->pnum);
  716. goto out_error;
  717. }
  718. vol_id = be32_to_cpu(vid_hdr->vol_id);
  719. lnum = be32_to_cpu(vid_hdr->lnum);
  720. err = ubi_eba_copy_leb(ubi, e1->pnum, e2->pnum, vid_hdr);
  721. if (err) {
  722. if (err == MOVE_CANCEL_RACE) {
  723. /*
  724. * The LEB has not been moved because the volume is
  725. * being deleted or the PEB has been put meanwhile. We
  726. * should prevent this PEB from being selected for
  727. * wear-leveling movement again, so put it to the
  728. * protection queue.
  729. */
  730. protect = 1;
  731. goto out_not_moved;
  732. }
  733. if (err == MOVE_RETRY) {
  734. scrubbing = 1;
  735. goto out_not_moved;
  736. }
  737. if (err == MOVE_TARGET_BITFLIPS || err == MOVE_TARGET_WR_ERR ||
  738. err == MOVE_TARGET_RD_ERR) {
  739. /*
  740. * Target PEB had bit-flips or write error - torture it.
  741. */
  742. torture = 1;
  743. goto out_not_moved;
  744. }
  745. if (err == MOVE_SOURCE_RD_ERR) {
  746. /*
  747. * An error happened while reading the source PEB. Do
  748. * not switch to R/O mode in this case, and give the
  749. * upper layers a possibility to recover from this,
  750. * e.g. by unmapping corresponding LEB. Instead, just
  751. * put this PEB to the @ubi->erroneous list to prevent
  752. * UBI from trying to move it over and over again.
  753. */
  754. if (ubi->erroneous_peb_count > ubi->max_erroneous) {
  755. ubi_err(ubi, "too many erroneous eraseblocks (%d)",
  756. ubi->erroneous_peb_count);
  757. goto out_error;
  758. }
  759. erroneous = 1;
  760. goto out_not_moved;
  761. }
  762. if (err < 0)
  763. goto out_error;
  764. ubi_assert(0);
  765. }
  766. /* The PEB has been successfully moved */
  767. if (scrubbing)
  768. ubi_msg(ubi, "scrubbed PEB %d (LEB %d:%d), data moved to PEB %d",
  769. e1->pnum, vol_id, lnum, e2->pnum);
  770. ubi_free_vid_hdr(ubi, vid_hdr);
  771. spin_lock(&ubi->wl_lock);
  772. if (!ubi->move_to_put) {
  773. wl_tree_add(e2, &ubi->used);
  774. e2 = NULL;
  775. }
  776. ubi->move_from = ubi->move_to = NULL;
  777. ubi->move_to_put = ubi->wl_scheduled = 0;
  778. spin_unlock(&ubi->wl_lock);
  779. err = do_sync_erase(ubi, e1, vol_id, lnum, 0);
  780. if (err) {
  781. if (e2)
  782. wl_entry_destroy(ubi, e2);
  783. goto out_ro;
  784. }
  785. if (e2) {
  786. /*
  787. * Well, the target PEB was put meanwhile, schedule it for
  788. * erasure.
  789. */
  790. dbg_wl("PEB %d (LEB %d:%d) was put meanwhile, erase",
  791. e2->pnum, vol_id, lnum);
  792. err = do_sync_erase(ubi, e2, vol_id, lnum, 0);
  793. if (err)
  794. goto out_ro;
  795. }
  796. dbg_wl("done");
  797. mutex_unlock(&ubi->move_mutex);
  798. return 0;
  799. /*
  800. * For some reasons the LEB was not moved, might be an error, might be
  801. * something else. @e1 was not changed, so return it back. @e2 might
  802. * have been changed, schedule it for erasure.
  803. */
  804. out_not_moved:
  805. if (vol_id != -1)
  806. dbg_wl("cancel moving PEB %d (LEB %d:%d) to PEB %d (%d)",
  807. e1->pnum, vol_id, lnum, e2->pnum, err);
  808. else
  809. dbg_wl("cancel moving PEB %d to PEB %d (%d)",
  810. e1->pnum, e2->pnum, err);
  811. spin_lock(&ubi->wl_lock);
  812. if (protect)
  813. prot_queue_add(ubi, e1);
  814. else if (erroneous) {
  815. wl_tree_add(e1, &ubi->erroneous);
  816. ubi->erroneous_peb_count += 1;
  817. } else if (scrubbing)
  818. wl_tree_add(e1, &ubi->scrub);
  819. else
  820. wl_tree_add(e1, &ubi->used);
  821. ubi_assert(!ubi->move_to_put);
  822. ubi->move_from = ubi->move_to = NULL;
  823. ubi->wl_scheduled = 0;
  824. spin_unlock(&ubi->wl_lock);
  825. ubi_free_vid_hdr(ubi, vid_hdr);
  826. err = do_sync_erase(ubi, e2, vol_id, lnum, torture);
  827. if (err)
  828. goto out_ro;
  829. mutex_unlock(&ubi->move_mutex);
  830. return 0;
  831. out_error:
  832. if (vol_id != -1)
  833. ubi_err(ubi, "error %d while moving PEB %d to PEB %d",
  834. err, e1->pnum, e2->pnum);
  835. else
  836. ubi_err(ubi, "error %d while moving PEB %d (LEB %d:%d) to PEB %d",
  837. err, e1->pnum, vol_id, lnum, e2->pnum);
  838. spin_lock(&ubi->wl_lock);
  839. ubi->move_from = ubi->move_to = NULL;
  840. ubi->move_to_put = ubi->wl_scheduled = 0;
  841. spin_unlock(&ubi->wl_lock);
  842. ubi_free_vid_hdr(ubi, vid_hdr);
  843. wl_entry_destroy(ubi, e1);
  844. wl_entry_destroy(ubi, e2);
  845. out_ro:
  846. ubi_ro_mode(ubi);
  847. mutex_unlock(&ubi->move_mutex);
  848. ubi_assert(err != 0);
  849. return err < 0 ? err : -EIO;
  850. out_cancel:
  851. ubi->wl_scheduled = 0;
  852. spin_unlock(&ubi->wl_lock);
  853. mutex_unlock(&ubi->move_mutex);
  854. ubi_free_vid_hdr(ubi, vid_hdr);
  855. return 0;
  856. }
  857. /**
  858. * ensure_wear_leveling - schedule wear-leveling if it is needed.
  859. * @ubi: UBI device description object
  860. * @nested: set to non-zero if this function is called from UBI worker
  861. *
  862. * This function checks if it is time to start wear-leveling and schedules it
  863. * if yes. This function returns zero in case of success and a negative error
  864. * code in case of failure.
  865. */
  866. static int ensure_wear_leveling(struct ubi_device *ubi, int nested)
  867. {
  868. int err = 0;
  869. struct ubi_wl_entry *e1;
  870. struct ubi_wl_entry *e2;
  871. struct ubi_work *wrk;
  872. spin_lock(&ubi->wl_lock);
  873. if (ubi->wl_scheduled)
  874. /* Wear-leveling is already in the work queue */
  875. goto out_unlock;
  876. /*
  877. * If the ubi->scrub tree is not empty, scrubbing is needed, and the
  878. * the WL worker has to be scheduled anyway.
  879. */
  880. if (!ubi->scrub.rb_node) {
  881. if (!ubi->used.rb_node || !ubi->free.rb_node)
  882. /* No physical eraseblocks - no deal */
  883. goto out_unlock;
  884. /*
  885. * We schedule wear-leveling only if the difference between the
  886. * lowest erase counter of used physical eraseblocks and a high
  887. * erase counter of free physical eraseblocks is greater than
  888. * %UBI_WL_THRESHOLD.
  889. */
  890. e1 = rb_entry(rb_first(&ubi->used), struct ubi_wl_entry, u.rb);
  891. e2 = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF);
  892. if (!(e2->ec - e1->ec >= UBI_WL_THRESHOLD))
  893. goto out_unlock;
  894. dbg_wl("schedule wear-leveling");
  895. } else
  896. dbg_wl("schedule scrubbing");
  897. ubi->wl_scheduled = 1;
  898. spin_unlock(&ubi->wl_lock);
  899. wrk = kmalloc(sizeof(struct ubi_work), GFP_NOFS);
  900. if (!wrk) {
  901. err = -ENOMEM;
  902. goto out_cancel;
  903. }
  904. wrk->anchor = 0;
  905. wrk->func = &wear_leveling_worker;
  906. if (nested)
  907. __schedule_ubi_work(ubi, wrk);
  908. #ifndef __UBOOT__
  909. else
  910. schedule_ubi_work(ubi, wrk);
  911. #else
  912. else {
  913. schedule_ubi_work(ubi, wrk);
  914. ubi_do_worker(ubi);
  915. }
  916. #endif
  917. return err;
  918. out_cancel:
  919. spin_lock(&ubi->wl_lock);
  920. ubi->wl_scheduled = 0;
  921. out_unlock:
  922. spin_unlock(&ubi->wl_lock);
  923. return err;
  924. }
  925. /**
  926. * erase_worker - physical eraseblock erase worker function.
  927. * @ubi: UBI device description object
  928. * @wl_wrk: the work object
  929. * @shutdown: non-zero if the worker has to free memory and exit
  930. * because the WL sub-system is shutting down
  931. *
  932. * This function erases a physical eraseblock and perform torture testing if
  933. * needed. It also takes care about marking the physical eraseblock bad if
  934. * needed. Returns zero in case of success and a negative error code in case of
  935. * failure.
  936. */
  937. static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk,
  938. int shutdown)
  939. {
  940. struct ubi_wl_entry *e = wl_wrk->e;
  941. int pnum = e->pnum;
  942. int vol_id = wl_wrk->vol_id;
  943. int lnum = wl_wrk->lnum;
  944. int err, available_consumed = 0;
  945. if (shutdown) {
  946. dbg_wl("cancel erasure of PEB %d EC %d", pnum, e->ec);
  947. kfree(wl_wrk);
  948. wl_entry_destroy(ubi, e);
  949. return 0;
  950. }
  951. dbg_wl("erase PEB %d EC %d LEB %d:%d",
  952. pnum, e->ec, wl_wrk->vol_id, wl_wrk->lnum);
  953. err = sync_erase(ubi, e, wl_wrk->torture);
  954. if (!err) {
  955. /* Fine, we've erased it successfully */
  956. kfree(wl_wrk);
  957. spin_lock(&ubi->wl_lock);
  958. wl_tree_add(e, &ubi->free);
  959. ubi->free_count++;
  960. spin_unlock(&ubi->wl_lock);
  961. /*
  962. * One more erase operation has happened, take care about
  963. * protected physical eraseblocks.
  964. */
  965. serve_prot_queue(ubi);
  966. /* And take care about wear-leveling */
  967. err = ensure_wear_leveling(ubi, 1);
  968. return err;
  969. }
  970. ubi_err(ubi, "failed to erase PEB %d, error %d", pnum, err);
  971. kfree(wl_wrk);
  972. if (err == -EINTR || err == -ENOMEM || err == -EAGAIN ||
  973. err == -EBUSY) {
  974. int err1;
  975. /* Re-schedule the LEB for erasure */
  976. err1 = schedule_erase(ubi, e, vol_id, lnum, 0);
  977. if (err1) {
  978. err = err1;
  979. goto out_ro;
  980. }
  981. return err;
  982. }
  983. wl_entry_destroy(ubi, e);
  984. if (err != -EIO)
  985. /*
  986. * If this is not %-EIO, we have no idea what to do. Scheduling
  987. * this physical eraseblock for erasure again would cause
  988. * errors again and again. Well, lets switch to R/O mode.
  989. */
  990. goto out_ro;
  991. /* It is %-EIO, the PEB went bad */
  992. if (!ubi->bad_allowed) {
  993. ubi_err(ubi, "bad physical eraseblock %d detected", pnum);
  994. goto out_ro;
  995. }
  996. spin_lock(&ubi->volumes_lock);
  997. if (ubi->beb_rsvd_pebs == 0) {
  998. if (ubi->avail_pebs == 0) {
  999. spin_unlock(&ubi->volumes_lock);
  1000. ubi_err(ubi, "no reserved/available physical eraseblocks");
  1001. goto out_ro;
  1002. }
  1003. ubi->avail_pebs -= 1;
  1004. available_consumed = 1;
  1005. }
  1006. spin_unlock(&ubi->volumes_lock);
  1007. ubi_msg(ubi, "mark PEB %d as bad", pnum);
  1008. err = ubi_io_mark_bad(ubi, pnum);
  1009. if (err)
  1010. goto out_ro;
  1011. spin_lock(&ubi->volumes_lock);
  1012. if (ubi->beb_rsvd_pebs > 0) {
  1013. if (available_consumed) {
  1014. /*
  1015. * The amount of reserved PEBs increased since we last
  1016. * checked.
  1017. */
  1018. ubi->avail_pebs += 1;
  1019. available_consumed = 0;
  1020. }
  1021. ubi->beb_rsvd_pebs -= 1;
  1022. }
  1023. ubi->bad_peb_count += 1;
  1024. ubi->good_peb_count -= 1;
  1025. ubi_calculate_reserved(ubi);
  1026. if (available_consumed)
  1027. ubi_warn(ubi, "no PEBs in the reserved pool, used an available PEB");
  1028. else if (ubi->beb_rsvd_pebs)
  1029. ubi_msg(ubi, "%d PEBs left in the reserve",
  1030. ubi->beb_rsvd_pebs);
  1031. else
  1032. ubi_warn(ubi, "last PEB from the reserve was used");
  1033. spin_unlock(&ubi->volumes_lock);
  1034. return err;
  1035. out_ro:
  1036. if (available_consumed) {
  1037. spin_lock(&ubi->volumes_lock);
  1038. ubi->avail_pebs += 1;
  1039. spin_unlock(&ubi->volumes_lock);
  1040. }
  1041. ubi_ro_mode(ubi);
  1042. return err;
  1043. }
  1044. /**
  1045. * ubi_wl_put_peb - return a PEB to the wear-leveling sub-system.
  1046. * @ubi: UBI device description object
  1047. * @vol_id: the volume ID that last used this PEB
  1048. * @lnum: the last used logical eraseblock number for the PEB
  1049. * @pnum: physical eraseblock to return
  1050. * @torture: if this physical eraseblock has to be tortured
  1051. *
  1052. * This function is called to return physical eraseblock @pnum to the pool of
  1053. * free physical eraseblocks. The @torture flag has to be set if an I/O error
  1054. * occurred to this @pnum and it has to be tested. This function returns zero
  1055. * in case of success, and a negative error code in case of failure.
  1056. */
  1057. int ubi_wl_put_peb(struct ubi_device *ubi, int vol_id, int lnum,
  1058. int pnum, int torture)
  1059. {
  1060. int err;
  1061. struct ubi_wl_entry *e;
  1062. dbg_wl("PEB %d", pnum);
  1063. ubi_assert(pnum >= 0);
  1064. ubi_assert(pnum < ubi->peb_count);
  1065. down_read(&ubi->fm_protect);
  1066. retry:
  1067. spin_lock(&ubi->wl_lock);
  1068. e = ubi->lookuptbl[pnum];
  1069. if (e == ubi->move_from) {
  1070. /*
  1071. * User is putting the physical eraseblock which was selected to
  1072. * be moved. It will be scheduled for erasure in the
  1073. * wear-leveling worker.
  1074. */
  1075. dbg_wl("PEB %d is being moved, wait", pnum);
  1076. spin_unlock(&ubi->wl_lock);
  1077. /* Wait for the WL worker by taking the @ubi->move_mutex */
  1078. mutex_lock(&ubi->move_mutex);
  1079. mutex_unlock(&ubi->move_mutex);
  1080. goto retry;
  1081. } else if (e == ubi->move_to) {
  1082. /*
  1083. * User is putting the physical eraseblock which was selected
  1084. * as the target the data is moved to. It may happen if the EBA
  1085. * sub-system already re-mapped the LEB in 'ubi_eba_copy_leb()'
  1086. * but the WL sub-system has not put the PEB to the "used" tree
  1087. * yet, but it is about to do this. So we just set a flag which
  1088. * will tell the WL worker that the PEB is not needed anymore
  1089. * and should be scheduled for erasure.
  1090. */
  1091. dbg_wl("PEB %d is the target of data moving", pnum);
  1092. ubi_assert(!ubi->move_to_put);
  1093. ubi->move_to_put = 1;
  1094. spin_unlock(&ubi->wl_lock);
  1095. up_read(&ubi->fm_protect);
  1096. return 0;
  1097. } else {
  1098. if (in_wl_tree(e, &ubi->used)) {
  1099. self_check_in_wl_tree(ubi, e, &ubi->used);
  1100. rb_erase(&e->u.rb, &ubi->used);
  1101. } else if (in_wl_tree(e, &ubi->scrub)) {
  1102. self_check_in_wl_tree(ubi, e, &ubi->scrub);
  1103. rb_erase(&e->u.rb, &ubi->scrub);
  1104. } else if (in_wl_tree(e, &ubi->erroneous)) {
  1105. self_check_in_wl_tree(ubi, e, &ubi->erroneous);
  1106. rb_erase(&e->u.rb, &ubi->erroneous);
  1107. ubi->erroneous_peb_count -= 1;
  1108. ubi_assert(ubi->erroneous_peb_count >= 0);
  1109. /* Erroneous PEBs should be tortured */
  1110. torture = 1;
  1111. } else {
  1112. err = prot_queue_del(ubi, e->pnum);
  1113. if (err) {
  1114. ubi_err(ubi, "PEB %d not found", pnum);
  1115. ubi_ro_mode(ubi);
  1116. spin_unlock(&ubi->wl_lock);
  1117. up_read(&ubi->fm_protect);
  1118. return err;
  1119. }
  1120. }
  1121. }
  1122. spin_unlock(&ubi->wl_lock);
  1123. err = schedule_erase(ubi, e, vol_id, lnum, torture);
  1124. if (err) {
  1125. spin_lock(&ubi->wl_lock);
  1126. wl_tree_add(e, &ubi->used);
  1127. spin_unlock(&ubi->wl_lock);
  1128. }
  1129. up_read(&ubi->fm_protect);
  1130. return err;
  1131. }
  1132. /**
  1133. * ubi_wl_scrub_peb - schedule a physical eraseblock for scrubbing.
  1134. * @ubi: UBI device description object
  1135. * @pnum: the physical eraseblock to schedule
  1136. *
  1137. * If a bit-flip in a physical eraseblock is detected, this physical eraseblock
  1138. * needs scrubbing. This function schedules a physical eraseblock for
  1139. * scrubbing which is done in background. This function returns zero in case of
  1140. * success and a negative error code in case of failure.
  1141. */
  1142. int ubi_wl_scrub_peb(struct ubi_device *ubi, int pnum)
  1143. {
  1144. struct ubi_wl_entry *e;
  1145. ubi_msg(ubi, "schedule PEB %d for scrubbing", pnum);
  1146. retry:
  1147. spin_lock(&ubi->wl_lock);
  1148. e = ubi->lookuptbl[pnum];
  1149. if (e == ubi->move_from || in_wl_tree(e, &ubi->scrub) ||
  1150. in_wl_tree(e, &ubi->erroneous)) {
  1151. spin_unlock(&ubi->wl_lock);
  1152. return 0;
  1153. }
  1154. if (e == ubi->move_to) {
  1155. /*
  1156. * This physical eraseblock was used to move data to. The data
  1157. * was moved but the PEB was not yet inserted to the proper
  1158. * tree. We should just wait a little and let the WL worker
  1159. * proceed.
  1160. */
  1161. spin_unlock(&ubi->wl_lock);
  1162. dbg_wl("the PEB %d is not in proper tree, retry", pnum);
  1163. yield();
  1164. goto retry;
  1165. }
  1166. if (in_wl_tree(e, &ubi->used)) {
  1167. self_check_in_wl_tree(ubi, e, &ubi->used);
  1168. rb_erase(&e->u.rb, &ubi->used);
  1169. } else {
  1170. int err;
  1171. err = prot_queue_del(ubi, e->pnum);
  1172. if (err) {
  1173. ubi_err(ubi, "PEB %d not found", pnum);
  1174. ubi_ro_mode(ubi);
  1175. spin_unlock(&ubi->wl_lock);
  1176. return err;
  1177. }
  1178. }
  1179. wl_tree_add(e, &ubi->scrub);
  1180. spin_unlock(&ubi->wl_lock);
  1181. /*
  1182. * Technically scrubbing is the same as wear-leveling, so it is done
  1183. * by the WL worker.
  1184. */
  1185. return ensure_wear_leveling(ubi, 0);
  1186. }
  1187. /**
  1188. * ubi_wl_flush - flush all pending works.
  1189. * @ubi: UBI device description object
  1190. * @vol_id: the volume id to flush for
  1191. * @lnum: the logical eraseblock number to flush for
  1192. *
  1193. * This function executes all pending works for a particular volume id /
  1194. * logical eraseblock number pair. If either value is set to %UBI_ALL, then it
  1195. * acts as a wildcard for all of the corresponding volume numbers or logical
  1196. * eraseblock numbers. It returns zero in case of success and a negative error
  1197. * code in case of failure.
  1198. */
  1199. int ubi_wl_flush(struct ubi_device *ubi, int vol_id, int lnum)
  1200. {
  1201. int err = 0;
  1202. int found = 1;
  1203. /*
  1204. * Erase while the pending works queue is not empty, but not more than
  1205. * the number of currently pending works.
  1206. */
  1207. dbg_wl("flush pending work for LEB %d:%d (%d pending works)",
  1208. vol_id, lnum, ubi->works_count);
  1209. while (found) {
  1210. struct ubi_work *wrk, *tmp;
  1211. found = 0;
  1212. down_read(&ubi->work_sem);
  1213. spin_lock(&ubi->wl_lock);
  1214. list_for_each_entry_safe(wrk, tmp, &ubi->works, list) {
  1215. if ((vol_id == UBI_ALL || wrk->vol_id == vol_id) &&
  1216. (lnum == UBI_ALL || wrk->lnum == lnum)) {
  1217. list_del(&wrk->list);
  1218. ubi->works_count -= 1;
  1219. ubi_assert(ubi->works_count >= 0);
  1220. spin_unlock(&ubi->wl_lock);
  1221. err = wrk->func(ubi, wrk, 0);
  1222. if (err) {
  1223. up_read(&ubi->work_sem);
  1224. return err;
  1225. }
  1226. spin_lock(&ubi->wl_lock);
  1227. found = 1;
  1228. break;
  1229. }
  1230. }
  1231. spin_unlock(&ubi->wl_lock);
  1232. up_read(&ubi->work_sem);
  1233. }
  1234. /*
  1235. * Make sure all the works which have been done in parallel are
  1236. * finished.
  1237. */
  1238. down_write(&ubi->work_sem);
  1239. up_write(&ubi->work_sem);
  1240. return err;
  1241. }
  1242. /**
  1243. * tree_destroy - destroy an RB-tree.
  1244. * @ubi: UBI device description object
  1245. * @root: the root of the tree to destroy
  1246. */
  1247. static void tree_destroy(struct ubi_device *ubi, struct rb_root *root)
  1248. {
  1249. struct rb_node *rb;
  1250. struct ubi_wl_entry *e;
  1251. rb = root->rb_node;
  1252. while (rb) {
  1253. if (rb->rb_left)
  1254. rb = rb->rb_left;
  1255. else if (rb->rb_right)
  1256. rb = rb->rb_right;
  1257. else {
  1258. e = rb_entry(rb, struct ubi_wl_entry, u.rb);
  1259. rb = rb_parent(rb);
  1260. if (rb) {
  1261. if (rb->rb_left == &e->u.rb)
  1262. rb->rb_left = NULL;
  1263. else
  1264. rb->rb_right = NULL;
  1265. }
  1266. wl_entry_destroy(ubi, e);
  1267. }
  1268. }
  1269. }
  1270. /**
  1271. * ubi_thread - UBI background thread.
  1272. * @u: the UBI device description object pointer
  1273. */
  1274. int ubi_thread(void *u)
  1275. {
  1276. int failures = 0;
  1277. struct ubi_device *ubi = u;
  1278. ubi_msg(ubi, "background thread \"%s\" started, PID %d",
  1279. ubi->bgt_name, task_pid_nr(current));
  1280. set_freezable();
  1281. for (;;) {
  1282. int err;
  1283. if (kthread_should_stop())
  1284. break;
  1285. if (try_to_freeze())
  1286. continue;
  1287. spin_lock(&ubi->wl_lock);
  1288. if (list_empty(&ubi->works) || ubi->ro_mode ||
  1289. !ubi->thread_enabled || ubi_dbg_is_bgt_disabled(ubi)) {
  1290. set_current_state(TASK_INTERRUPTIBLE);
  1291. spin_unlock(&ubi->wl_lock);
  1292. schedule();
  1293. continue;
  1294. }
  1295. spin_unlock(&ubi->wl_lock);
  1296. err = do_work(ubi);
  1297. if (err) {
  1298. ubi_err(ubi, "%s: work failed with error code %d",
  1299. ubi->bgt_name, err);
  1300. if (failures++ > WL_MAX_FAILURES) {
  1301. /*
  1302. * Too many failures, disable the thread and
  1303. * switch to read-only mode.
  1304. */
  1305. ubi_msg(ubi, "%s: %d consecutive failures",
  1306. ubi->bgt_name, WL_MAX_FAILURES);
  1307. ubi_ro_mode(ubi);
  1308. ubi->thread_enabled = 0;
  1309. continue;
  1310. }
  1311. } else
  1312. failures = 0;
  1313. cond_resched();
  1314. }
  1315. dbg_wl("background thread \"%s\" is killed", ubi->bgt_name);
  1316. return 0;
  1317. }
  1318. /**
  1319. * shutdown_work - shutdown all pending works.
  1320. * @ubi: UBI device description object
  1321. */
  1322. static void shutdown_work(struct ubi_device *ubi)
  1323. {
  1324. #ifdef CONFIG_MTD_UBI_FASTMAP
  1325. #ifndef __UBOOT__
  1326. flush_work(&ubi->fm_work);
  1327. #else
  1328. /* in U-Boot, we have all work done */
  1329. #endif
  1330. #endif
  1331. while (!list_empty(&ubi->works)) {
  1332. struct ubi_work *wrk;
  1333. wrk = list_entry(ubi->works.next, struct ubi_work, list);
  1334. list_del(&wrk->list);
  1335. wrk->func(ubi, wrk, 1);
  1336. ubi->works_count -= 1;
  1337. ubi_assert(ubi->works_count >= 0);
  1338. }
  1339. }
  1340. /**
  1341. * ubi_wl_init - initialize the WL sub-system using attaching information.
  1342. * @ubi: UBI device description object
  1343. * @ai: attaching information
  1344. *
  1345. * This function returns zero in case of success, and a negative error code in
  1346. * case of failure.
  1347. */
  1348. int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
  1349. {
  1350. int err, i, reserved_pebs, found_pebs = 0;
  1351. struct rb_node *rb1, *rb2;
  1352. struct ubi_ainf_volume *av;
  1353. struct ubi_ainf_peb *aeb, *tmp;
  1354. struct ubi_wl_entry *e;
  1355. ubi->used = ubi->erroneous = ubi->free = ubi->scrub = RB_ROOT;
  1356. spin_lock_init(&ubi->wl_lock);
  1357. mutex_init(&ubi->move_mutex);
  1358. init_rwsem(&ubi->work_sem);
  1359. ubi->max_ec = ai->max_ec;
  1360. INIT_LIST_HEAD(&ubi->works);
  1361. sprintf(ubi->bgt_name, UBI_BGT_NAME_PATTERN, ubi->ubi_num);
  1362. err = -ENOMEM;
  1363. ubi->lookuptbl = kzalloc(ubi->peb_count * sizeof(void *), GFP_KERNEL);
  1364. if (!ubi->lookuptbl)
  1365. return err;
  1366. for (i = 0; i < UBI_PROT_QUEUE_LEN; i++)
  1367. INIT_LIST_HEAD(&ubi->pq[i]);
  1368. ubi->pq_head = 0;
  1369. ubi->free_count = 0;
  1370. list_for_each_entry_safe(aeb, tmp, &ai->erase, u.list) {
  1371. cond_resched();
  1372. e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
  1373. if (!e)
  1374. goto out_free;
  1375. e->pnum = aeb->pnum;
  1376. e->ec = aeb->ec;
  1377. ubi->lookuptbl[e->pnum] = e;
  1378. if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0)) {
  1379. wl_entry_destroy(ubi, e);
  1380. goto out_free;
  1381. }
  1382. found_pebs++;
  1383. }
  1384. list_for_each_entry(aeb, &ai->free, u.list) {
  1385. cond_resched();
  1386. e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
  1387. if (!e)
  1388. goto out_free;
  1389. e->pnum = aeb->pnum;
  1390. e->ec = aeb->ec;
  1391. ubi_assert(e->ec >= 0);
  1392. wl_tree_add(e, &ubi->free);
  1393. ubi->free_count++;
  1394. ubi->lookuptbl[e->pnum] = e;
  1395. found_pebs++;
  1396. }
  1397. ubi_rb_for_each_entry(rb1, av, &ai->volumes, rb) {
  1398. ubi_rb_for_each_entry(rb2, aeb, &av->root, u.rb) {
  1399. cond_resched();
  1400. e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
  1401. if (!e)
  1402. goto out_free;
  1403. e->pnum = aeb->pnum;
  1404. e->ec = aeb->ec;
  1405. ubi->lookuptbl[e->pnum] = e;
  1406. if (!aeb->scrub) {
  1407. dbg_wl("add PEB %d EC %d to the used tree",
  1408. e->pnum, e->ec);
  1409. wl_tree_add(e, &ubi->used);
  1410. } else {
  1411. dbg_wl("add PEB %d EC %d to the scrub tree",
  1412. e->pnum, e->ec);
  1413. wl_tree_add(e, &ubi->scrub);
  1414. }
  1415. found_pebs++;
  1416. }
  1417. }
  1418. dbg_wl("found %i PEBs", found_pebs);
  1419. if (ubi->fm) {
  1420. ubi_assert(ubi->good_peb_count ==
  1421. found_pebs + ubi->fm->used_blocks);
  1422. for (i = 0; i < ubi->fm->used_blocks; i++) {
  1423. e = ubi->fm->e[i];
  1424. ubi->lookuptbl[e->pnum] = e;
  1425. }
  1426. }
  1427. else
  1428. ubi_assert(ubi->good_peb_count == found_pebs);
  1429. reserved_pebs = WL_RESERVED_PEBS;
  1430. ubi_fastmap_init(ubi, &reserved_pebs);
  1431. if (ubi->avail_pebs < reserved_pebs) {
  1432. ubi_err(ubi, "no enough physical eraseblocks (%d, need %d)",
  1433. ubi->avail_pebs, reserved_pebs);
  1434. if (ubi->corr_peb_count)
  1435. ubi_err(ubi, "%d PEBs are corrupted and not used",
  1436. ubi->corr_peb_count);
  1437. goto out_free;
  1438. }
  1439. ubi->avail_pebs -= reserved_pebs;
  1440. ubi->rsvd_pebs += reserved_pebs;
  1441. /* Schedule wear-leveling if needed */
  1442. err = ensure_wear_leveling(ubi, 0);
  1443. if (err)
  1444. goto out_free;
  1445. return 0;
  1446. out_free:
  1447. shutdown_work(ubi);
  1448. tree_destroy(ubi, &ubi->used);
  1449. tree_destroy(ubi, &ubi->free);
  1450. tree_destroy(ubi, &ubi->scrub);
  1451. kfree(ubi->lookuptbl);
  1452. return err;
  1453. }
  1454. /**
  1455. * protection_queue_destroy - destroy the protection queue.
  1456. * @ubi: UBI device description object
  1457. */
  1458. static void protection_queue_destroy(struct ubi_device *ubi)
  1459. {
  1460. int i;
  1461. struct ubi_wl_entry *e, *tmp;
  1462. for (i = 0; i < UBI_PROT_QUEUE_LEN; ++i) {
  1463. list_for_each_entry_safe(e, tmp, &ubi->pq[i], u.list) {
  1464. list_del(&e->u.list);
  1465. wl_entry_destroy(ubi, e);
  1466. }
  1467. }
  1468. }
  1469. /**
  1470. * ubi_wl_close - close the wear-leveling sub-system.
  1471. * @ubi: UBI device description object
  1472. */
  1473. void ubi_wl_close(struct ubi_device *ubi)
  1474. {
  1475. dbg_wl("close the WL sub-system");
  1476. ubi_fastmap_close(ubi);
  1477. shutdown_work(ubi);
  1478. protection_queue_destroy(ubi);
  1479. tree_destroy(ubi, &ubi->used);
  1480. tree_destroy(ubi, &ubi->erroneous);
  1481. tree_destroy(ubi, &ubi->free);
  1482. tree_destroy(ubi, &ubi->scrub);
  1483. kfree(ubi->lookuptbl);
  1484. }
  1485. /**
  1486. * self_check_ec - make sure that the erase counter of a PEB is correct.
  1487. * @ubi: UBI device description object
  1488. * @pnum: the physical eraseblock number to check
  1489. * @ec: the erase counter to check
  1490. *
  1491. * This function returns zero if the erase counter of physical eraseblock @pnum
  1492. * is equivalent to @ec, and a negative error code if not or if an error
  1493. * occurred.
  1494. */
  1495. static int self_check_ec(struct ubi_device *ubi, int pnum, int ec)
  1496. {
  1497. int err;
  1498. long long read_ec;
  1499. struct ubi_ec_hdr *ec_hdr;
  1500. if (!ubi_dbg_chk_gen(ubi))
  1501. return 0;
  1502. ec_hdr = kzalloc(ubi->ec_hdr_alsize, GFP_NOFS);
  1503. if (!ec_hdr)
  1504. return -ENOMEM;
  1505. err = ubi_io_read_ec_hdr(ubi, pnum, ec_hdr, 0);
  1506. if (err && err != UBI_IO_BITFLIPS) {
  1507. /* The header does not have to exist */
  1508. err = 0;
  1509. goto out_free;
  1510. }
  1511. read_ec = be64_to_cpu(ec_hdr->ec);
  1512. if (ec != read_ec && read_ec - ec > 1) {
  1513. ubi_err(ubi, "self-check failed for PEB %d", pnum);
  1514. ubi_err(ubi, "read EC is %lld, should be %d", read_ec, ec);
  1515. dump_stack();
  1516. err = 1;
  1517. } else
  1518. err = 0;
  1519. out_free:
  1520. kfree(ec_hdr);
  1521. return err;
  1522. }
  1523. /**
  1524. * self_check_in_wl_tree - check that wear-leveling entry is in WL RB-tree.
  1525. * @ubi: UBI device description object
  1526. * @e: the wear-leveling entry to check
  1527. * @root: the root of the tree
  1528. *
  1529. * This function returns zero if @e is in the @root RB-tree and %-EINVAL if it
  1530. * is not.
  1531. */
  1532. static int self_check_in_wl_tree(const struct ubi_device *ubi,
  1533. struct ubi_wl_entry *e, struct rb_root *root)
  1534. {
  1535. if (!ubi_dbg_chk_gen(ubi))
  1536. return 0;
  1537. if (in_wl_tree(e, root))
  1538. return 0;
  1539. ubi_err(ubi, "self-check failed for PEB %d, EC %d, RB-tree %p ",
  1540. e->pnum, e->ec, root);
  1541. dump_stack();
  1542. return -EINVAL;
  1543. }
  1544. /**
  1545. * self_check_in_pq - check if wear-leveling entry is in the protection
  1546. * queue.
  1547. * @ubi: UBI device description object
  1548. * @e: the wear-leveling entry to check
  1549. *
  1550. * This function returns zero if @e is in @ubi->pq and %-EINVAL if it is not.
  1551. */
  1552. static int self_check_in_pq(const struct ubi_device *ubi,
  1553. struct ubi_wl_entry *e)
  1554. {
  1555. struct ubi_wl_entry *p;
  1556. int i;
  1557. if (!ubi_dbg_chk_gen(ubi))
  1558. return 0;
  1559. for (i = 0; i < UBI_PROT_QUEUE_LEN; ++i)
  1560. list_for_each_entry(p, &ubi->pq[i], u.list)
  1561. if (p == e)
  1562. return 0;
  1563. ubi_err(ubi, "self-check failed for PEB %d, EC %d, Protect queue",
  1564. e->pnum, e->ec);
  1565. dump_stack();
  1566. return -EINVAL;
  1567. }
  1568. #ifndef CONFIG_MTD_UBI_FASTMAP
  1569. static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi)
  1570. {
  1571. struct ubi_wl_entry *e;
  1572. e = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF);
  1573. self_check_in_wl_tree(ubi, e, &ubi->free);
  1574. ubi->free_count--;
  1575. ubi_assert(ubi->free_count >= 0);
  1576. rb_erase(&e->u.rb, &ubi->free);
  1577. return e;
  1578. }
  1579. /**
  1580. * produce_free_peb - produce a free physical eraseblock.
  1581. * @ubi: UBI device description object
  1582. *
  1583. * This function tries to make a free PEB by means of synchronous execution of
  1584. * pending works. This may be needed if, for example the background thread is
  1585. * disabled. Returns zero in case of success and a negative error code in case
  1586. * of failure.
  1587. */
  1588. static int produce_free_peb(struct ubi_device *ubi)
  1589. {
  1590. int err;
  1591. while (!ubi->free.rb_node && ubi->works_count) {
  1592. spin_unlock(&ubi->wl_lock);
  1593. dbg_wl("do one work synchronously");
  1594. err = do_work(ubi);
  1595. spin_lock(&ubi->wl_lock);
  1596. if (err)
  1597. return err;
  1598. }
  1599. return 0;
  1600. }
  1601. /**
  1602. * ubi_wl_get_peb - get a physical eraseblock.
  1603. * @ubi: UBI device description object
  1604. *
  1605. * This function returns a physical eraseblock in case of success and a
  1606. * negative error code in case of failure.
  1607. * Returns with ubi->fm_eba_sem held in read mode!
  1608. */
  1609. int ubi_wl_get_peb(struct ubi_device *ubi)
  1610. {
  1611. int err;
  1612. struct ubi_wl_entry *e;
  1613. retry:
  1614. down_read(&ubi->fm_eba_sem);
  1615. spin_lock(&ubi->wl_lock);
  1616. if (!ubi->free.rb_node) {
  1617. if (ubi->works_count == 0) {
  1618. ubi_err(ubi, "no free eraseblocks");
  1619. ubi_assert(list_empty(&ubi->works));
  1620. spin_unlock(&ubi->wl_lock);
  1621. return -ENOSPC;
  1622. }
  1623. err = produce_free_peb(ubi);
  1624. if (err < 0) {
  1625. spin_unlock(&ubi->wl_lock);
  1626. return err;
  1627. }
  1628. spin_unlock(&ubi->wl_lock);
  1629. up_read(&ubi->fm_eba_sem);
  1630. goto retry;
  1631. }
  1632. e = wl_get_wle(ubi);
  1633. prot_queue_add(ubi, e);
  1634. spin_unlock(&ubi->wl_lock);
  1635. err = ubi_self_check_all_ff(ubi, e->pnum, ubi->vid_hdr_aloffset,
  1636. ubi->peb_size - ubi->vid_hdr_aloffset);
  1637. if (err) {
  1638. ubi_err(ubi, "new PEB %d does not contain all 0xFF bytes", e->pnum);
  1639. return err;
  1640. }
  1641. return e->pnum;
  1642. }
  1643. #else
  1644. #include "fastmap-wl.c"
  1645. #endif