runqueue.py 119 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940
  1. """
  2. BitBake 'RunQueue' implementation
  3. Handles preparation and execution of a queue of tasks
  4. """
  5. # Copyright (C) 2006-2007 Richard Purdie
  6. #
  7. # SPDX-License-Identifier: GPL-2.0-only
  8. #
  9. import copy
  10. import os
  11. import sys
  12. import signal
  13. import stat
  14. import fcntl
  15. import errno
  16. import logging
  17. import re
  18. import bb
  19. from bb import msg, data, event
  20. from bb import monitordisk
  21. import subprocess
  22. import pickle
  23. from multiprocessing import Process
  24. import shlex
  25. import pprint
  26. bblogger = logging.getLogger("BitBake")
  27. logger = logging.getLogger("BitBake.RunQueue")
  28. __find_sha256__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{64}(?![a-z0-9])' )
  29. def fn_from_tid(tid):
  30. return tid.rsplit(":", 1)[0]
  31. def taskname_from_tid(tid):
  32. return tid.rsplit(":", 1)[1]
  33. def mc_from_tid(tid):
  34. if tid.startswith('mc:'):
  35. return tid.split(':')[1]
  36. return ""
  37. def split_tid(tid):
  38. (mc, fn, taskname, _) = split_tid_mcfn(tid)
  39. return (mc, fn, taskname)
  40. def split_tid_mcfn(tid):
  41. if tid.startswith('mc:'):
  42. elems = tid.split(':')
  43. mc = elems[1]
  44. fn = ":".join(elems[2:-1])
  45. taskname = elems[-1]
  46. mcfn = "mc:" + mc + ":" + fn
  47. else:
  48. tid = tid.rsplit(":", 1)
  49. mc = ""
  50. fn = tid[0]
  51. taskname = tid[1]
  52. mcfn = fn
  53. return (mc, fn, taskname, mcfn)
  54. def build_tid(mc, fn, taskname):
  55. if mc:
  56. return "mc:" + mc + ":" + fn + ":" + taskname
  57. return fn + ":" + taskname
  58. # Index used to pair up potentially matching multiconfig tasks
  59. # We match on PN, taskname and hash being equal
  60. def pending_hash_index(tid, rqdata):
  61. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  62. pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
  63. h = rqdata.runtaskentries[tid].hash
  64. return pn + ":" + "taskname" + h
  65. class RunQueueStats:
  66. """
  67. Holds statistics on the tasks handled by the associated runQueue
  68. """
  69. def __init__(self, total):
  70. self.completed = 0
  71. self.skipped = 0
  72. self.failed = 0
  73. self.active = 0
  74. self.total = total
  75. def copy(self):
  76. obj = self.__class__(self.total)
  77. obj.__dict__.update(self.__dict__)
  78. return obj
  79. def taskFailed(self):
  80. self.active = self.active - 1
  81. self.failed = self.failed + 1
  82. def taskCompleted(self):
  83. self.active = self.active - 1
  84. self.completed = self.completed + 1
  85. def taskSkipped(self):
  86. self.active = self.active + 1
  87. self.skipped = self.skipped + 1
  88. def taskActive(self):
  89. self.active = self.active + 1
  90. # These values indicate the next step due to be run in the
  91. # runQueue state machine
  92. runQueuePrepare = 2
  93. runQueueSceneInit = 3
  94. runQueueRunning = 6
  95. runQueueFailed = 7
  96. runQueueCleanUp = 8
  97. runQueueComplete = 9
  98. class RunQueueScheduler(object):
  99. """
  100. Control the order tasks are scheduled in.
  101. """
  102. name = "basic"
  103. def __init__(self, runqueue, rqdata):
  104. """
  105. The default scheduler just returns the first buildable task (the
  106. priority map is sorted by task number)
  107. """
  108. self.rq = runqueue
  109. self.rqdata = rqdata
  110. self.numTasks = len(self.rqdata.runtaskentries)
  111. self.prio_map = [self.rqdata.runtaskentries.keys()]
  112. self.buildable = set()
  113. self.skip_maxthread = {}
  114. self.stamps = {}
  115. for tid in self.rqdata.runtaskentries:
  116. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  117. self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
  118. if tid in self.rq.runq_buildable:
  119. self.buildable.append(tid)
  120. self.rev_prio_map = None
  121. def next_buildable_task(self):
  122. """
  123. Return the id of the first task we find that is buildable
  124. """
  125. buildable = set(self.buildable)
  126. buildable.difference_update(self.rq.runq_running)
  127. buildable.difference_update(self.rq.holdoff_tasks)
  128. buildable.intersection_update(self.rq.tasks_covered | self.rq.tasks_notcovered)
  129. if not buildable:
  130. return None
  131. # Filter out tasks that have a max number of threads that have been exceeded
  132. skip_buildable = {}
  133. for running in self.rq.runq_running.difference(self.rq.runq_complete):
  134. rtaskname = taskname_from_tid(running)
  135. if rtaskname not in self.skip_maxthread:
  136. self.skip_maxthread[rtaskname] = self.rq.cfgData.getVarFlag(rtaskname, "number_threads")
  137. if not self.skip_maxthread[rtaskname]:
  138. continue
  139. if rtaskname in skip_buildable:
  140. skip_buildable[rtaskname] += 1
  141. else:
  142. skip_buildable[rtaskname] = 1
  143. if len(buildable) == 1:
  144. tid = buildable.pop()
  145. taskname = taskname_from_tid(tid)
  146. if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
  147. return None
  148. stamp = self.stamps[tid]
  149. if stamp not in self.rq.build_stamps.values():
  150. return tid
  151. if not self.rev_prio_map:
  152. self.rev_prio_map = {}
  153. for tid in self.rqdata.runtaskentries:
  154. self.rev_prio_map[tid] = self.prio_map.index(tid)
  155. best = None
  156. bestprio = None
  157. for tid in buildable:
  158. taskname = taskname_from_tid(tid)
  159. if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
  160. continue
  161. prio = self.rev_prio_map[tid]
  162. if bestprio is None or bestprio > prio:
  163. stamp = self.stamps[tid]
  164. if stamp in self.rq.build_stamps.values():
  165. continue
  166. bestprio = prio
  167. best = tid
  168. return best
  169. def next(self):
  170. """
  171. Return the id of the task we should build next
  172. """
  173. if self.rq.can_start_task():
  174. return self.next_buildable_task()
  175. def newbuildable(self, task):
  176. self.buildable.add(task)
  177. def removebuildable(self, task):
  178. self.buildable.remove(task)
  179. def describe_task(self, taskid):
  180. result = 'ID %s' % taskid
  181. if self.rev_prio_map:
  182. result = result + (' pri %d' % self.rev_prio_map[taskid])
  183. return result
  184. def dump_prio(self, comment):
  185. bb.debug(3, '%s (most important first):\n%s' %
  186. (comment,
  187. '\n'.join(['%d. %s' % (index + 1, self.describe_task(taskid)) for
  188. index, taskid in enumerate(self.prio_map)])))
  189. class RunQueueSchedulerSpeed(RunQueueScheduler):
  190. """
  191. A scheduler optimised for speed. The priority map is sorted by task weight,
  192. heavier weighted tasks (tasks needed by the most other tasks) are run first.
  193. """
  194. name = "speed"
  195. def __init__(self, runqueue, rqdata):
  196. """
  197. The priority map is sorted by task weight.
  198. """
  199. RunQueueScheduler.__init__(self, runqueue, rqdata)
  200. weights = {}
  201. for tid in self.rqdata.runtaskentries:
  202. weight = self.rqdata.runtaskentries[tid].weight
  203. if not weight in weights:
  204. weights[weight] = []
  205. weights[weight].append(tid)
  206. self.prio_map = []
  207. for weight in sorted(weights):
  208. for w in weights[weight]:
  209. self.prio_map.append(w)
  210. self.prio_map.reverse()
  211. class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
  212. """
  213. A scheduler optimised to complete .bb files as quickly as possible. The
  214. priority map is sorted by task weight, but then reordered so once a given
  215. .bb file starts to build, it's completed as quickly as possible by
  216. running all tasks related to the same .bb file one after the after.
  217. This works well where disk space is at a premium and classes like OE's
  218. rm_work are in force.
  219. """
  220. name = "completion"
  221. def __init__(self, runqueue, rqdata):
  222. super(RunQueueSchedulerCompletion, self).__init__(runqueue, rqdata)
  223. # Extract list of tasks for each recipe, with tasks sorted
  224. # ascending from "must run first" (typically do_fetch) to
  225. # "runs last" (do_build). The speed scheduler prioritizes
  226. # tasks that must run first before the ones that run later;
  227. # this is what we depend on here.
  228. task_lists = {}
  229. for taskid in self.prio_map:
  230. fn, taskname = taskid.rsplit(':', 1)
  231. task_lists.setdefault(fn, []).append(taskname)
  232. # Now unify the different task lists. The strategy is that
  233. # common tasks get skipped and new ones get inserted after the
  234. # preceeding common one(s) as they are found. Because task
  235. # lists should differ only by their number of tasks, but not
  236. # the ordering of the common tasks, this should result in a
  237. # deterministic result that is a superset of the individual
  238. # task ordering.
  239. all_tasks = []
  240. for recipe, new_tasks in task_lists.items():
  241. index = 0
  242. old_task = all_tasks[index] if index < len(all_tasks) else None
  243. for new_task in new_tasks:
  244. if old_task == new_task:
  245. # Common task, skip it. This is the fast-path which
  246. # avoids a full search.
  247. index += 1
  248. old_task = all_tasks[index] if index < len(all_tasks) else None
  249. else:
  250. try:
  251. index = all_tasks.index(new_task)
  252. # Already present, just not at the current
  253. # place. We re-synchronized by changing the
  254. # index so that it matches again. Now
  255. # move on to the next existing task.
  256. index += 1
  257. old_task = all_tasks[index] if index < len(all_tasks) else None
  258. except ValueError:
  259. # Not present. Insert before old_task, which
  260. # remains the same (but gets shifted back).
  261. all_tasks.insert(index, new_task)
  262. index += 1
  263. bb.debug(3, 'merged task list: %s' % all_tasks)
  264. # Now reverse the order so that tasks that finish the work on one
  265. # recipe are considered more imporant (= come first). The ordering
  266. # is now so that do_build is most important.
  267. all_tasks.reverse()
  268. # Group tasks of the same kind before tasks of less important
  269. # kinds at the head of the queue (because earlier = lower
  270. # priority number = runs earlier), while preserving the
  271. # ordering by recipe. If recipe foo is more important than
  272. # bar, then the goal is to work on foo's do_populate_sysroot
  273. # before bar's do_populate_sysroot and on the more important
  274. # tasks of foo before any of the less important tasks in any
  275. # other recipe (if those other recipes are more important than
  276. # foo).
  277. #
  278. # All of this only applies when tasks are runable. Explicit
  279. # dependencies still override this ordering by priority.
  280. #
  281. # Here's an example why this priority re-ordering helps with
  282. # minimizing disk usage. Consider a recipe foo with a higher
  283. # priority than bar where foo DEPENDS on bar. Then the
  284. # implicit rule (from base.bbclass) is that foo's do_configure
  285. # depends on bar's do_populate_sysroot. This ensures that
  286. # bar's do_populate_sysroot gets done first. Normally the
  287. # tasks from foo would continue to run once that is done, and
  288. # bar only gets completed and cleaned up later. By ordering
  289. # bar's task that depend on bar's do_populate_sysroot before foo's
  290. # do_configure, that problem gets avoided.
  291. task_index = 0
  292. self.dump_prio('original priorities')
  293. for task in all_tasks:
  294. for index in range(task_index, self.numTasks):
  295. taskid = self.prio_map[index]
  296. taskname = taskid.rsplit(':', 1)[1]
  297. if taskname == task:
  298. del self.prio_map[index]
  299. self.prio_map.insert(task_index, taskid)
  300. task_index += 1
  301. self.dump_prio('completion priorities')
  302. class RunTaskEntry(object):
  303. def __init__(self):
  304. self.depends = set()
  305. self.revdeps = set()
  306. self.hash = None
  307. self.unihash = None
  308. self.task = None
  309. self.weight = 1
  310. class RunQueueData:
  311. """
  312. BitBake Run Queue implementation
  313. """
  314. def __init__(self, rq, cooker, cfgData, dataCaches, taskData, targets):
  315. self.cooker = cooker
  316. self.dataCaches = dataCaches
  317. self.taskData = taskData
  318. self.targets = targets
  319. self.rq = rq
  320. self.warn_multi_bb = False
  321. self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST") or ""
  322. self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split()
  323. self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
  324. self.setscenewhitelist_checked = False
  325. self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1")
  326. self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
  327. self.reset()
  328. def reset(self):
  329. self.runtaskentries = {}
  330. def runq_depends_names(self, ids):
  331. import re
  332. ret = []
  333. for id in ids:
  334. nam = os.path.basename(id)
  335. nam = re.sub("_[^,]*,", ",", nam)
  336. ret.extend([nam])
  337. return ret
  338. def get_task_hash(self, tid):
  339. return self.runtaskentries[tid].hash
  340. def get_task_unihash(self, tid):
  341. return self.runtaskentries[tid].unihash
  342. def get_user_idstring(self, tid, task_name_suffix = ""):
  343. return tid + task_name_suffix
  344. def get_short_user_idstring(self, task, task_name_suffix = ""):
  345. (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
  346. pn = self.dataCaches[mc].pkg_fn[taskfn]
  347. taskname = taskname_from_tid(task) + task_name_suffix
  348. return "%s:%s" % (pn, taskname)
  349. def circular_depchains_handler(self, tasks):
  350. """
  351. Some tasks aren't buildable, likely due to circular dependency issues.
  352. Identify the circular dependencies and print them in a user readable format.
  353. """
  354. from copy import deepcopy
  355. valid_chains = []
  356. explored_deps = {}
  357. msgs = []
  358. class TooManyLoops(Exception):
  359. pass
  360. def chain_reorder(chain):
  361. """
  362. Reorder a dependency chain so the lowest task id is first
  363. """
  364. lowest = 0
  365. new_chain = []
  366. for entry in range(len(chain)):
  367. if chain[entry] < chain[lowest]:
  368. lowest = entry
  369. new_chain.extend(chain[lowest:])
  370. new_chain.extend(chain[:lowest])
  371. return new_chain
  372. def chain_compare_equal(chain1, chain2):
  373. """
  374. Compare two dependency chains and see if they're the same
  375. """
  376. if len(chain1) != len(chain2):
  377. return False
  378. for index in range(len(chain1)):
  379. if chain1[index] != chain2[index]:
  380. return False
  381. return True
  382. def chain_array_contains(chain, chain_array):
  383. """
  384. Return True if chain_array contains chain
  385. """
  386. for ch in chain_array:
  387. if chain_compare_equal(ch, chain):
  388. return True
  389. return False
  390. def find_chains(tid, prev_chain):
  391. prev_chain.append(tid)
  392. total_deps = []
  393. total_deps.extend(self.runtaskentries[tid].revdeps)
  394. for revdep in self.runtaskentries[tid].revdeps:
  395. if revdep in prev_chain:
  396. idx = prev_chain.index(revdep)
  397. # To prevent duplicates, reorder the chain to start with the lowest taskid
  398. # and search through an array of those we've already printed
  399. chain = prev_chain[idx:]
  400. new_chain = chain_reorder(chain)
  401. if not chain_array_contains(new_chain, valid_chains):
  402. valid_chains.append(new_chain)
  403. msgs.append("Dependency loop #%d found:\n" % len(valid_chains))
  404. for dep in new_chain:
  405. msgs.append(" Task %s (dependent Tasks %s)\n" % (dep, self.runq_depends_names(self.runtaskentries[dep].depends)))
  406. msgs.append("\n")
  407. if len(valid_chains) > 10:
  408. msgs.append("Aborted dependency loops search after 10 matches.\n")
  409. raise TooManyLoops
  410. continue
  411. scan = False
  412. if revdep not in explored_deps:
  413. scan = True
  414. elif revdep in explored_deps[revdep]:
  415. scan = True
  416. else:
  417. for dep in prev_chain:
  418. if dep in explored_deps[revdep]:
  419. scan = True
  420. if scan:
  421. find_chains(revdep, copy.deepcopy(prev_chain))
  422. for dep in explored_deps[revdep]:
  423. if dep not in total_deps:
  424. total_deps.append(dep)
  425. explored_deps[tid] = total_deps
  426. try:
  427. for task in tasks:
  428. find_chains(task, [])
  429. except TooManyLoops:
  430. pass
  431. return msgs
  432. def calculate_task_weights(self, endpoints):
  433. """
  434. Calculate a number representing the "weight" of each task. Heavier weighted tasks
  435. have more dependencies and hence should be executed sooner for maximum speed.
  436. This function also sanity checks the task list finding tasks that are not
  437. possible to execute due to circular dependencies.
  438. """
  439. numTasks = len(self.runtaskentries)
  440. weight = {}
  441. deps_left = {}
  442. task_done = {}
  443. for tid in self.runtaskentries:
  444. task_done[tid] = False
  445. weight[tid] = 1
  446. deps_left[tid] = len(self.runtaskentries[tid].revdeps)
  447. for tid in endpoints:
  448. weight[tid] = 10
  449. task_done[tid] = True
  450. while True:
  451. next_points = []
  452. for tid in endpoints:
  453. for revdep in self.runtaskentries[tid].depends:
  454. weight[revdep] = weight[revdep] + weight[tid]
  455. deps_left[revdep] = deps_left[revdep] - 1
  456. if deps_left[revdep] == 0:
  457. next_points.append(revdep)
  458. task_done[revdep] = True
  459. endpoints = next_points
  460. if len(next_points) == 0:
  461. break
  462. # Circular dependency sanity check
  463. problem_tasks = []
  464. for tid in self.runtaskentries:
  465. if task_done[tid] is False or deps_left[tid] != 0:
  466. problem_tasks.append(tid)
  467. logger.debug(2, "Task %s is not buildable", tid)
  468. logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid])
  469. self.runtaskentries[tid].weight = weight[tid]
  470. if problem_tasks:
  471. message = "%s unbuildable tasks were found.\n" % len(problem_tasks)
  472. message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n"
  473. message = message + "Identifying dependency loops (this may take a short while)...\n"
  474. logger.error(message)
  475. msgs = self.circular_depchains_handler(problem_tasks)
  476. message = "\n"
  477. for msg in msgs:
  478. message = message + msg
  479. bb.msg.fatal("RunQueue", message)
  480. return weight
  481. def prepare(self):
  482. """
  483. Turn a set of taskData into a RunQueue and compute data needed
  484. to optimise the execution order.
  485. """
  486. runq_build = {}
  487. recursivetasks = {}
  488. recursiveitasks = {}
  489. recursivetasksselfref = set()
  490. taskData = self.taskData
  491. found = False
  492. for mc in self.taskData:
  493. if len(taskData[mc].taskentries) > 0:
  494. found = True
  495. break
  496. if not found:
  497. # Nothing to do
  498. return 0
  499. self.init_progress_reporter.start()
  500. self.init_progress_reporter.next_stage()
  501. # Step A - Work out a list of tasks to run
  502. #
  503. # Taskdata gives us a list of possible providers for every build and run
  504. # target ordered by priority. It also gives information on each of those
  505. # providers.
  506. #
  507. # To create the actual list of tasks to execute we fix the list of
  508. # providers and then resolve the dependencies into task IDs. This
  509. # process is repeated for each type of dependency (tdepends, deptask,
  510. # rdeptast, recrdeptask, idepends).
  511. def add_build_dependencies(depids, tasknames, depends, mc):
  512. for depname in depids:
  513. # Won't be in build_targets if ASSUME_PROVIDED
  514. if depname not in taskData[mc].build_targets or not taskData[mc].build_targets[depname]:
  515. continue
  516. depdata = taskData[mc].build_targets[depname][0]
  517. if depdata is None:
  518. continue
  519. for taskname in tasknames:
  520. t = depdata + ":" + taskname
  521. if t in taskData[mc].taskentries:
  522. depends.add(t)
  523. def add_runtime_dependencies(depids, tasknames, depends, mc):
  524. for depname in depids:
  525. if depname not in taskData[mc].run_targets or not taskData[mc].run_targets[depname]:
  526. continue
  527. depdata = taskData[mc].run_targets[depname][0]
  528. if depdata is None:
  529. continue
  530. for taskname in tasknames:
  531. t = depdata + ":" + taskname
  532. if t in taskData[mc].taskentries:
  533. depends.add(t)
  534. def add_mc_dependencies(mc, tid):
  535. mcdeps = taskData[mc].get_mcdepends()
  536. for dep in mcdeps:
  537. mcdependency = dep.split(':')
  538. pn = mcdependency[3]
  539. frommc = mcdependency[1]
  540. mcdep = mcdependency[2]
  541. deptask = mcdependency[4]
  542. if mc == frommc:
  543. fn = taskData[mcdep].build_targets[pn][0]
  544. newdep = '%s:%s' % (fn,deptask)
  545. taskData[mc].taskentries[tid].tdepends.append(newdep)
  546. for mc in taskData:
  547. for tid in taskData[mc].taskentries:
  548. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  549. #runtid = build_tid(mc, fn, taskname)
  550. #logger.debug(2, "Processing %s,%s:%s", mc, fn, taskname)
  551. depends = set()
  552. task_deps = self.dataCaches[mc].task_deps[taskfn]
  553. self.runtaskentries[tid] = RunTaskEntry()
  554. if fn in taskData[mc].failed_fns:
  555. continue
  556. # We add multiconfig dependencies before processing internal task deps (tdepends)
  557. if 'mcdepends' in task_deps and taskname in task_deps['mcdepends']:
  558. add_mc_dependencies(mc, tid)
  559. # Resolve task internal dependencies
  560. #
  561. # e.g. addtask before X after Y
  562. for t in taskData[mc].taskentries[tid].tdepends:
  563. (depmc, depfn, deptaskname, _) = split_tid_mcfn(t)
  564. depends.add(build_tid(depmc, depfn, deptaskname))
  565. # Resolve 'deptask' dependencies
  566. #
  567. # e.g. do_sometask[deptask] = "do_someothertask"
  568. # (makes sure sometask runs after someothertask of all DEPENDS)
  569. if 'deptask' in task_deps and taskname in task_deps['deptask']:
  570. tasknames = task_deps['deptask'][taskname].split()
  571. add_build_dependencies(taskData[mc].depids[taskfn], tasknames, depends, mc)
  572. # Resolve 'rdeptask' dependencies
  573. #
  574. # e.g. do_sometask[rdeptask] = "do_someothertask"
  575. # (makes sure sometask runs after someothertask of all RDEPENDS)
  576. if 'rdeptask' in task_deps and taskname in task_deps['rdeptask']:
  577. tasknames = task_deps['rdeptask'][taskname].split()
  578. add_runtime_dependencies(taskData[mc].rdepids[taskfn], tasknames, depends, mc)
  579. # Resolve inter-task dependencies
  580. #
  581. # e.g. do_sometask[depends] = "targetname:do_someothertask"
  582. # (makes sure sometask runs after targetname's someothertask)
  583. idepends = taskData[mc].taskentries[tid].idepends
  584. for (depname, idependtask) in idepends:
  585. if depname in taskData[mc].build_targets and taskData[mc].build_targets[depname] and not depname in taskData[mc].failed_deps:
  586. # Won't be in build_targets if ASSUME_PROVIDED
  587. depdata = taskData[mc].build_targets[depname][0]
  588. if depdata is not None:
  589. t = depdata + ":" + idependtask
  590. depends.add(t)
  591. if t not in taskData[mc].taskentries:
  592. bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata))
  593. irdepends = taskData[mc].taskentries[tid].irdepends
  594. for (depname, idependtask) in irdepends:
  595. if depname in taskData[mc].run_targets:
  596. # Won't be in run_targets if ASSUME_PROVIDED
  597. if not taskData[mc].run_targets[depname]:
  598. continue
  599. depdata = taskData[mc].run_targets[depname][0]
  600. if depdata is not None:
  601. t = depdata + ":" + idependtask
  602. depends.add(t)
  603. if t not in taskData[mc].taskentries:
  604. bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata))
  605. # Resolve recursive 'recrdeptask' dependencies (Part A)
  606. #
  607. # e.g. do_sometask[recrdeptask] = "do_someothertask"
  608. # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
  609. # We cover the recursive part of the dependencies below
  610. if 'recrdeptask' in task_deps and taskname in task_deps['recrdeptask']:
  611. tasknames = task_deps['recrdeptask'][taskname].split()
  612. recursivetasks[tid] = tasknames
  613. add_build_dependencies(taskData[mc].depids[taskfn], tasknames, depends, mc)
  614. add_runtime_dependencies(taskData[mc].rdepids[taskfn], tasknames, depends, mc)
  615. if taskname in tasknames:
  616. recursivetasksselfref.add(tid)
  617. if 'recideptask' in task_deps and taskname in task_deps['recideptask']:
  618. recursiveitasks[tid] = []
  619. for t in task_deps['recideptask'][taskname].split():
  620. newdep = build_tid(mc, fn, t)
  621. recursiveitasks[tid].append(newdep)
  622. self.runtaskentries[tid].depends = depends
  623. # Remove all self references
  624. self.runtaskentries[tid].depends.discard(tid)
  625. #self.dump_data()
  626. self.init_progress_reporter.next_stage()
  627. # Resolve recursive 'recrdeptask' dependencies (Part B)
  628. #
  629. # e.g. do_sometask[recrdeptask] = "do_someothertask"
  630. # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
  631. # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
  632. # Generating/interating recursive lists of dependencies is painful and potentially slow
  633. # Precompute recursive task dependencies here by:
  634. # a) create a temp list of reverse dependencies (revdeps)
  635. # b) walk up the ends of the chains (when a given task no longer has dependencies i.e. len(deps) == 0)
  636. # c) combine the total list of dependencies in cumulativedeps
  637. # d) optimise by pre-truncating 'task' off the items in cumulativedeps (keeps items in sets lower)
  638. revdeps = {}
  639. deps = {}
  640. cumulativedeps = {}
  641. for tid in self.runtaskentries:
  642. deps[tid] = set(self.runtaskentries[tid].depends)
  643. revdeps[tid] = set()
  644. cumulativedeps[tid] = set()
  645. # Generate a temp list of reverse dependencies
  646. for tid in self.runtaskentries:
  647. for dep in self.runtaskentries[tid].depends:
  648. revdeps[dep].add(tid)
  649. # Find the dependency chain endpoints
  650. endpoints = set()
  651. for tid in self.runtaskentries:
  652. if len(deps[tid]) == 0:
  653. endpoints.add(tid)
  654. # Iterate the chains collating dependencies
  655. while endpoints:
  656. next = set()
  657. for tid in endpoints:
  658. for dep in revdeps[tid]:
  659. cumulativedeps[dep].add(fn_from_tid(tid))
  660. cumulativedeps[dep].update(cumulativedeps[tid])
  661. if tid in deps[dep]:
  662. deps[dep].remove(tid)
  663. if len(deps[dep]) == 0:
  664. next.add(dep)
  665. endpoints = next
  666. #for tid in deps:
  667. # if len(deps[tid]) != 0:
  668. # bb.warn("Sanity test failure, dependencies left for %s (%s)" % (tid, deps[tid]))
  669. # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to
  670. # resolve these recursively until we aren't adding any further extra dependencies
  671. extradeps = True
  672. while extradeps:
  673. extradeps = 0
  674. for tid in recursivetasks:
  675. tasknames = recursivetasks[tid]
  676. totaldeps = set(self.runtaskentries[tid].depends)
  677. if tid in recursiveitasks:
  678. totaldeps.update(recursiveitasks[tid])
  679. for dep in recursiveitasks[tid]:
  680. if dep not in self.runtaskentries:
  681. continue
  682. totaldeps.update(self.runtaskentries[dep].depends)
  683. deps = set()
  684. for dep in totaldeps:
  685. if dep in cumulativedeps:
  686. deps.update(cumulativedeps[dep])
  687. for t in deps:
  688. for taskname in tasknames:
  689. newtid = t + ":" + taskname
  690. if newtid == tid:
  691. continue
  692. if newtid in self.runtaskentries and newtid not in self.runtaskentries[tid].depends:
  693. extradeps += 1
  694. self.runtaskentries[tid].depends.add(newtid)
  695. # Handle recursive tasks which depend upon other recursive tasks
  696. deps = set()
  697. for dep in self.runtaskentries[tid].depends.intersection(recursivetasks):
  698. deps.update(self.runtaskentries[dep].depends.difference(self.runtaskentries[tid].depends))
  699. for newtid in deps:
  700. for taskname in tasknames:
  701. if not newtid.endswith(":" + taskname):
  702. continue
  703. if newtid in self.runtaskentries:
  704. extradeps += 1
  705. self.runtaskentries[tid].depends.add(newtid)
  706. bb.debug(1, "Added %s recursive dependencies in this loop" % extradeps)
  707. # Remove recrdeptask circular references so that do_a[recrdeptask] = "do_a do_b" can work
  708. for tid in recursivetasksselfref:
  709. self.runtaskentries[tid].depends.difference_update(recursivetasksselfref)
  710. self.init_progress_reporter.next_stage()
  711. #self.dump_data()
  712. # Step B - Mark all active tasks
  713. #
  714. # Start with the tasks we were asked to run and mark all dependencies
  715. # as active too. If the task is to be 'forced', clear its stamp. Once
  716. # all active tasks are marked, prune the ones we don't need.
  717. logger.verbose("Marking Active Tasks")
  718. def mark_active(tid, depth):
  719. """
  720. Mark an item as active along with its depends
  721. (calls itself recursively)
  722. """
  723. if tid in runq_build:
  724. return
  725. runq_build[tid] = 1
  726. depends = self.runtaskentries[tid].depends
  727. for depend in depends:
  728. mark_active(depend, depth+1)
  729. def invalidate_task(tid, error_nostamp):
  730. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  731. taskdep = self.dataCaches[mc].task_deps[taskfn]
  732. if fn + ":" + taskname not in taskData[mc].taskentries:
  733. logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname)
  734. if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
  735. if error_nostamp:
  736. bb.fatal("Task %s is marked nostamp, cannot invalidate this task" % taskname)
  737. else:
  738. bb.debug(1, "Task %s is marked nostamp, cannot invalidate this task" % taskname)
  739. else:
  740. logger.verbose("Invalidate task %s, %s", taskname, fn)
  741. bb.parse.siggen.invalidate_task(taskname, self.dataCaches[mc], taskfn)
  742. self.target_tids = []
  743. for (mc, target, task, fn) in self.targets:
  744. if target not in taskData[mc].build_targets or not taskData[mc].build_targets[target]:
  745. continue
  746. if target in taskData[mc].failed_deps:
  747. continue
  748. parents = False
  749. if task.endswith('-'):
  750. parents = True
  751. task = task[:-1]
  752. if fn in taskData[mc].failed_fns:
  753. continue
  754. # fn already has mc prefix
  755. tid = fn + ":" + task
  756. self.target_tids.append(tid)
  757. if tid not in taskData[mc].taskentries:
  758. import difflib
  759. tasks = []
  760. for x in taskData[mc].taskentries:
  761. if x.startswith(fn + ":"):
  762. tasks.append(taskname_from_tid(x))
  763. close_matches = difflib.get_close_matches(task, tasks, cutoff=0.7)
  764. if close_matches:
  765. extra = ". Close matches:\n %s" % "\n ".join(close_matches)
  766. else:
  767. extra = ""
  768. bb.msg.fatal("RunQueue", "Task %s does not exist for target %s (%s)%s" % (task, target, tid, extra))
  769. # For tasks called "XXXX-", ony run their dependencies
  770. if parents:
  771. for i in self.runtaskentries[tid].depends:
  772. mark_active(i, 1)
  773. else:
  774. mark_active(tid, 1)
  775. self.init_progress_reporter.next_stage()
  776. # Step C - Prune all inactive tasks
  777. #
  778. # Once all active tasks are marked, prune the ones we don't need.
  779. delcount = {}
  780. for tid in list(self.runtaskentries.keys()):
  781. if tid not in runq_build:
  782. delcount[tid] = self.runtaskentries[tid]
  783. del self.runtaskentries[tid]
  784. # Handle --runall
  785. if self.cooker.configuration.runall:
  786. # re-run the mark_active and then drop unused tasks from new list
  787. runq_build = {}
  788. for task in self.cooker.configuration.runall:
  789. runall_tids = set()
  790. for tid in list(self.runtaskentries):
  791. wanttid = fn_from_tid(tid) + ":do_%s" % task
  792. if wanttid in delcount:
  793. self.runtaskentries[wanttid] = delcount[wanttid]
  794. if wanttid in self.runtaskentries:
  795. runall_tids.add(wanttid)
  796. for tid in list(runall_tids):
  797. mark_active(tid,1)
  798. if self.cooker.configuration.force:
  799. invalidate_task(tid, False)
  800. for tid in list(self.runtaskentries.keys()):
  801. if tid not in runq_build:
  802. delcount[tid] = self.runtaskentries[tid]
  803. del self.runtaskentries[tid]
  804. if len(self.runtaskentries) == 0:
  805. bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
  806. self.init_progress_reporter.next_stage()
  807. # Handle runonly
  808. if self.cooker.configuration.runonly:
  809. # re-run the mark_active and then drop unused tasks from new list
  810. runq_build = {}
  811. for task in self.cooker.configuration.runonly:
  812. runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == "do_%s" % task }
  813. for tid in list(runonly_tids):
  814. mark_active(tid,1)
  815. if self.cooker.configuration.force:
  816. invalidate_task(tid, False)
  817. for tid in list(self.runtaskentries.keys()):
  818. if tid not in runq_build:
  819. delcount[tid] = self.runtaskentries[tid]
  820. del self.runtaskentries[tid]
  821. if len(self.runtaskentries) == 0:
  822. bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the taskgraphs of the targets %s" % (str(self.cooker.configuration.runonly), str(self.targets)))
  823. #
  824. # Step D - Sanity checks and computation
  825. #
  826. # Check to make sure we still have tasks to run
  827. if len(self.runtaskentries) == 0:
  828. if not taskData[''].abort:
  829. bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.")
  830. else:
  831. bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.")
  832. logger.verbose("Pruned %s inactive tasks, %s left", len(delcount), len(self.runtaskentries))
  833. logger.verbose("Assign Weightings")
  834. self.init_progress_reporter.next_stage()
  835. # Generate a list of reverse dependencies to ease future calculations
  836. for tid in self.runtaskentries:
  837. for dep in self.runtaskentries[tid].depends:
  838. self.runtaskentries[dep].revdeps.add(tid)
  839. self.init_progress_reporter.next_stage()
  840. # Identify tasks at the end of dependency chains
  841. # Error on circular dependency loops (length two)
  842. endpoints = []
  843. for tid in self.runtaskentries:
  844. revdeps = self.runtaskentries[tid].revdeps
  845. if len(revdeps) == 0:
  846. endpoints.append(tid)
  847. for dep in revdeps:
  848. if dep in self.runtaskentries[tid].depends:
  849. bb.msg.fatal("RunQueue", "Task %s has circular dependency on %s" % (tid, dep))
  850. logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
  851. self.init_progress_reporter.next_stage()
  852. # Calculate task weights
  853. # Check of higher length circular dependencies
  854. self.runq_weight = self.calculate_task_weights(endpoints)
  855. self.init_progress_reporter.next_stage()
  856. # Sanity Check - Check for multiple tasks building the same provider
  857. for mc in self.dataCaches:
  858. prov_list = {}
  859. seen_fn = []
  860. for tid in self.runtaskentries:
  861. (tidmc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  862. if taskfn in seen_fn:
  863. continue
  864. if mc != tidmc:
  865. continue
  866. seen_fn.append(taskfn)
  867. for prov in self.dataCaches[mc].fn_provides[taskfn]:
  868. if prov not in prov_list:
  869. prov_list[prov] = [taskfn]
  870. elif taskfn not in prov_list[prov]:
  871. prov_list[prov].append(taskfn)
  872. for prov in prov_list:
  873. if len(prov_list[prov]) < 2:
  874. continue
  875. if prov in self.multi_provider_whitelist:
  876. continue
  877. seen_pn = []
  878. # If two versions of the same PN are being built its fatal, we don't support it.
  879. for fn in prov_list[prov]:
  880. pn = self.dataCaches[mc].pkg_fn[fn]
  881. if pn not in seen_pn:
  882. seen_pn.append(pn)
  883. else:
  884. bb.fatal("Multiple versions of %s are due to be built (%s). Only one version of a given PN should be built in any given build. You likely need to set PREFERRED_VERSION_%s to select the correct version or don't depend on multiple versions." % (pn, " ".join(prov_list[prov]), pn))
  885. msg = "Multiple .bb files are due to be built which each provide %s:\n %s" % (prov, "\n ".join(prov_list[prov]))
  886. #
  887. # Construct a list of things which uniquely depend on each provider
  888. # since this may help the user figure out which dependency is triggering this warning
  889. #
  890. msg += "\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from."
  891. deplist = {}
  892. commondeps = None
  893. for provfn in prov_list[prov]:
  894. deps = set()
  895. for tid in self.runtaskentries:
  896. fn = fn_from_tid(tid)
  897. if fn != provfn:
  898. continue
  899. for dep in self.runtaskentries[tid].revdeps:
  900. fn = fn_from_tid(dep)
  901. if fn == provfn:
  902. continue
  903. deps.add(dep)
  904. if not commondeps:
  905. commondeps = set(deps)
  906. else:
  907. commondeps &= deps
  908. deplist[provfn] = deps
  909. for provfn in deplist:
  910. msg += "\n%s has unique dependees:\n %s" % (provfn, "\n ".join(deplist[provfn] - commondeps))
  911. #
  912. # Construct a list of provides and runtime providers for each recipe
  913. # (rprovides has to cover RPROVIDES, PACKAGES, PACKAGES_DYNAMIC)
  914. #
  915. msg += "\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful."
  916. provide_results = {}
  917. rprovide_results = {}
  918. commonprovs = None
  919. commonrprovs = None
  920. for provfn in prov_list[prov]:
  921. provides = set(self.dataCaches[mc].fn_provides[provfn])
  922. rprovides = set()
  923. for rprovide in self.dataCaches[mc].rproviders:
  924. if provfn in self.dataCaches[mc].rproviders[rprovide]:
  925. rprovides.add(rprovide)
  926. for package in self.dataCaches[mc].packages:
  927. if provfn in self.dataCaches[mc].packages[package]:
  928. rprovides.add(package)
  929. for package in self.dataCaches[mc].packages_dynamic:
  930. if provfn in self.dataCaches[mc].packages_dynamic[package]:
  931. rprovides.add(package)
  932. if not commonprovs:
  933. commonprovs = set(provides)
  934. else:
  935. commonprovs &= provides
  936. provide_results[provfn] = provides
  937. if not commonrprovs:
  938. commonrprovs = set(rprovides)
  939. else:
  940. commonrprovs &= rprovides
  941. rprovide_results[provfn] = rprovides
  942. #msg += "\nCommon provides:\n %s" % ("\n ".join(commonprovs))
  943. #msg += "\nCommon rprovides:\n %s" % ("\n ".join(commonrprovs))
  944. for provfn in prov_list[prov]:
  945. msg += "\n%s has unique provides:\n %s" % (provfn, "\n ".join(provide_results[provfn] - commonprovs))
  946. msg += "\n%s has unique rprovides:\n %s" % (provfn, "\n ".join(rprovide_results[provfn] - commonrprovs))
  947. if self.warn_multi_bb:
  948. logger.verbnote(msg)
  949. else:
  950. logger.error(msg)
  951. self.init_progress_reporter.next_stage()
  952. # Create a whitelist usable by the stamp checks
  953. self.stampfnwhitelist = {}
  954. for mc in self.taskData:
  955. self.stampfnwhitelist[mc] = []
  956. for entry in self.stampwhitelist.split():
  957. if entry not in self.taskData[mc].build_targets:
  958. continue
  959. fn = self.taskData.build_targets[entry][0]
  960. self.stampfnwhitelist[mc].append(fn)
  961. self.init_progress_reporter.next_stage()
  962. # Iterate over the task list looking for tasks with a 'setscene' function
  963. self.runq_setscene_tids = []
  964. if not self.cooker.configuration.nosetscene:
  965. for tid in self.runtaskentries:
  966. (mc, fn, taskname, _) = split_tid_mcfn(tid)
  967. setscenetid = tid + "_setscene"
  968. if setscenetid not in taskData[mc].taskentries:
  969. continue
  970. self.runq_setscene_tids.append(tid)
  971. self.init_progress_reporter.next_stage()
  972. # Invalidate task if force mode active
  973. if self.cooker.configuration.force:
  974. for tid in self.target_tids:
  975. invalidate_task(tid, False)
  976. # Invalidate task if invalidate mode active
  977. if self.cooker.configuration.invalidate_stamp:
  978. for tid in self.target_tids:
  979. fn = fn_from_tid(tid)
  980. for st in self.cooker.configuration.invalidate_stamp.split(','):
  981. if not st.startswith("do_"):
  982. st = "do_%s" % st
  983. invalidate_task(fn + ":" + st, True)
  984. self.init_progress_reporter.next_stage()
  985. # Create and print to the logs a virtual/xxxx -> PN (fn) table
  986. for mc in taskData:
  987. virtmap = taskData[mc].get_providermap(prefix="virtual/")
  988. virtpnmap = {}
  989. for v in virtmap:
  990. virtpnmap[v] = self.dataCaches[mc].pkg_fn[virtmap[v]]
  991. bb.debug(2, "%s resolved to: %s (%s)" % (v, virtpnmap[v], virtmap[v]))
  992. if hasattr(bb.parse.siggen, "tasks_resolved"):
  993. bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCaches[mc])
  994. self.init_progress_reporter.next_stage()
  995. # Iterate over the task list and call into the siggen code
  996. dealtwith = set()
  997. todeal = set(self.runtaskentries)
  998. while len(todeal) > 0:
  999. for tid in todeal.copy():
  1000. if len(self.runtaskentries[tid].depends - dealtwith) == 0:
  1001. dealtwith.add(tid)
  1002. todeal.remove(tid)
  1003. self.prepare_task_hash(tid)
  1004. bb.parse.siggen.writeout_file_checksum_cache()
  1005. bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
  1006. #self.dump_data()
  1007. return len(self.runtaskentries)
  1008. def prepare_task_hash(self, tid):
  1009. procdep = []
  1010. for dep in self.runtaskentries[tid].depends:
  1011. procdep.append(dep)
  1012. self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.dataCaches[mc_from_tid(tid)])
  1013. self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
  1014. def dump_data(self):
  1015. """
  1016. Dump some debug information on the internal data structures
  1017. """
  1018. logger.debug(3, "run_tasks:")
  1019. for tid in self.runtaskentries:
  1020. logger.debug(3, " %s: %s Deps %s RevDeps %s", tid,
  1021. self.runtaskentries[tid].weight,
  1022. self.runtaskentries[tid].depends,
  1023. self.runtaskentries[tid].revdeps)
  1024. class RunQueueWorker():
  1025. def __init__(self, process, pipe):
  1026. self.process = process
  1027. self.pipe = pipe
  1028. class RunQueue:
  1029. def __init__(self, cooker, cfgData, dataCaches, taskData, targets):
  1030. self.cooker = cooker
  1031. self.cfgData = cfgData
  1032. self.rqdata = RunQueueData(self, cooker, cfgData, dataCaches, taskData, targets)
  1033. self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY") or "perfile"
  1034. self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None
  1035. self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None
  1036. self.state = runQueuePrepare
  1037. # For disk space monitor
  1038. # Invoked at regular time intervals via the bitbake heartbeat event
  1039. # while the build is running. We generate a unique name for the handler
  1040. # here, just in case that there ever is more than one RunQueue instance,
  1041. # start the handler when reaching runQueueSceneInit, and stop it when
  1042. # done with the build.
  1043. self.dm = monitordisk.diskMonitor(cfgData)
  1044. self.dm_event_handler_name = '_bb_diskmonitor_' + str(id(self))
  1045. self.dm_event_handler_registered = False
  1046. self.rqexe = None
  1047. self.worker = {}
  1048. self.fakeworker = {}
  1049. def _start_worker(self, mc, fakeroot = False, rqexec = None):
  1050. logger.debug(1, "Starting bitbake-worker")
  1051. magic = "decafbad"
  1052. if self.cooker.configuration.profile:
  1053. magic = "decafbadbad"
  1054. if fakeroot:
  1055. magic = magic + "beef"
  1056. mcdata = self.cooker.databuilder.mcdata[mc]
  1057. fakerootcmd = shlex.split(mcdata.getVar("FAKEROOTCMD"))
  1058. fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split()
  1059. env = os.environ.copy()
  1060. for key, value in (var.split('=') for var in fakerootenv):
  1061. env[key] = value
  1062. worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
  1063. else:
  1064. worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
  1065. bb.utils.nonblockingfd(worker.stdout)
  1066. workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
  1067. workerdata = {
  1068. "taskdeps" : self.rqdata.dataCaches[mc].task_deps,
  1069. "fakerootenv" : self.rqdata.dataCaches[mc].fakerootenv,
  1070. "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs,
  1071. "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv,
  1072. "sigdata" : bb.parse.siggen.get_taskdata(),
  1073. "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
  1074. "logdefaultverbose" : bb.msg.loggerDefaultVerbose,
  1075. "logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
  1076. "logdefaultdomain" : bb.msg.loggerDefaultDomains,
  1077. "prhost" : self.cooker.prhost,
  1078. "buildname" : self.cfgData.getVar("BUILDNAME"),
  1079. "date" : self.cfgData.getVar("DATE"),
  1080. "time" : self.cfgData.getVar("TIME"),
  1081. "hashservaddr" : self.cooker.hashservaddr,
  1082. }
  1083. worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
  1084. worker.stdin.write(b"<extraconfigdata>" + pickle.dumps(self.cooker.extraconfigdata) + b"</extraconfigdata>")
  1085. worker.stdin.write(b"<workerdata>" + pickle.dumps(workerdata) + b"</workerdata>")
  1086. worker.stdin.flush()
  1087. return RunQueueWorker(worker, workerpipe)
  1088. def _teardown_worker(self, worker):
  1089. if not worker:
  1090. return
  1091. logger.debug(1, "Teardown for bitbake-worker")
  1092. try:
  1093. worker.process.stdin.write(b"<quit></quit>")
  1094. worker.process.stdin.flush()
  1095. worker.process.stdin.close()
  1096. except IOError:
  1097. pass
  1098. while worker.process.returncode is None:
  1099. worker.pipe.read()
  1100. worker.process.poll()
  1101. while worker.pipe.read():
  1102. continue
  1103. worker.pipe.close()
  1104. def start_worker(self):
  1105. if self.worker:
  1106. self.teardown_workers()
  1107. self.teardown = False
  1108. for mc in self.rqdata.dataCaches:
  1109. self.worker[mc] = self._start_worker(mc)
  1110. def start_fakeworker(self, rqexec, mc):
  1111. if not mc in self.fakeworker:
  1112. self.fakeworker[mc] = self._start_worker(mc, True, rqexec)
  1113. def teardown_workers(self):
  1114. self.teardown = True
  1115. for mc in self.worker:
  1116. self._teardown_worker(self.worker[mc])
  1117. self.worker = {}
  1118. for mc in self.fakeworker:
  1119. self._teardown_worker(self.fakeworker[mc])
  1120. self.fakeworker = {}
  1121. def read_workers(self):
  1122. for mc in self.worker:
  1123. self.worker[mc].pipe.read()
  1124. for mc in self.fakeworker:
  1125. self.fakeworker[mc].pipe.read()
  1126. def active_fds(self):
  1127. fds = []
  1128. for mc in self.worker:
  1129. fds.append(self.worker[mc].pipe.input)
  1130. for mc in self.fakeworker:
  1131. fds.append(self.fakeworker[mc].pipe.input)
  1132. return fds
  1133. def check_stamp_task(self, tid, taskname = None, recurse = False, cache = None):
  1134. def get_timestamp(f):
  1135. try:
  1136. if not os.access(f, os.F_OK):
  1137. return None
  1138. return os.stat(f)[stat.ST_MTIME]
  1139. except:
  1140. return None
  1141. (mc, fn, tn, taskfn) = split_tid_mcfn(tid)
  1142. if taskname is None:
  1143. taskname = tn
  1144. if self.stamppolicy == "perfile":
  1145. fulldeptree = False
  1146. else:
  1147. fulldeptree = True
  1148. stampwhitelist = []
  1149. if self.stamppolicy == "whitelist":
  1150. stampwhitelist = self.rqdata.stampfnwhitelist[mc]
  1151. stampfile = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn)
  1152. # If the stamp is missing, it's not current
  1153. if not os.access(stampfile, os.F_OK):
  1154. logger.debug(2, "Stampfile %s not available", stampfile)
  1155. return False
  1156. # If it's a 'nostamp' task, it's not current
  1157. taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
  1158. if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
  1159. logger.debug(2, "%s.%s is nostamp\n", fn, taskname)
  1160. return False
  1161. if taskname != "do_setscene" and taskname.endswith("_setscene"):
  1162. return True
  1163. if cache is None:
  1164. cache = {}
  1165. iscurrent = True
  1166. t1 = get_timestamp(stampfile)
  1167. for dep in self.rqdata.runtaskentries[tid].depends:
  1168. if iscurrent:
  1169. (mc2, fn2, taskname2, taskfn2) = split_tid_mcfn(dep)
  1170. stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCaches[mc2], taskfn2)
  1171. stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCaches[mc2], taskfn2)
  1172. t2 = get_timestamp(stampfile2)
  1173. t3 = get_timestamp(stampfile3)
  1174. if t3 and not t2:
  1175. continue
  1176. if t3 and t3 > t2:
  1177. continue
  1178. if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist):
  1179. if not t2:
  1180. logger.debug(2, 'Stampfile %s does not exist', stampfile2)
  1181. iscurrent = False
  1182. break
  1183. if t1 < t2:
  1184. logger.debug(2, 'Stampfile %s < %s', stampfile, stampfile2)
  1185. iscurrent = False
  1186. break
  1187. if recurse and iscurrent:
  1188. if dep in cache:
  1189. iscurrent = cache[dep]
  1190. if not iscurrent:
  1191. logger.debug(2, 'Stampfile for dependency %s:%s invalid (cached)' % (fn2, taskname2))
  1192. else:
  1193. iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache)
  1194. cache[dep] = iscurrent
  1195. if recurse:
  1196. cache[tid] = iscurrent
  1197. return iscurrent
  1198. def validate_hashes(self, tocheck, data, currentcount=None, siginfo=False):
  1199. valid = set()
  1200. if self.hashvalidate:
  1201. sq_data = {}
  1202. sq_data['hash'] = {}
  1203. sq_data['hashfn'] = {}
  1204. sq_data['unihash'] = {}
  1205. for tid in tocheck:
  1206. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  1207. sq_data['hash'][tid] = self.rqdata.runtaskentries[tid].hash
  1208. sq_data['hashfn'][tid] = self.rqdata.dataCaches[mc].hashfn[taskfn]
  1209. sq_data['unihash'][tid] = self.rqdata.runtaskentries[tid].unihash
  1210. valid = self.validate_hash(sq_data, data, siginfo, currentcount)
  1211. return valid
  1212. def validate_hash(self, sq_data, d, siginfo, currentcount):
  1213. locs = {"sq_data" : sq_data, "d" : d, "siginfo" : siginfo, "currentcount" : currentcount}
  1214. # Metadata has **kwargs so args can be added, sq_data can also gain new fields
  1215. call = self.hashvalidate + "(sq_data, d, siginfo=siginfo, currentcount=currentcount)"
  1216. return bb.utils.better_eval(call, locs)
  1217. def _execute_runqueue(self):
  1218. """
  1219. Run the tasks in a queue prepared by rqdata.prepare()
  1220. Upon failure, optionally try to recover the build using any alternate providers
  1221. (if the abort on failure configuration option isn't set)
  1222. """
  1223. retval = True
  1224. if self.state is runQueuePrepare:
  1225. # NOTE: if you add, remove or significantly refactor the stages of this
  1226. # process then you should recalculate the weightings here. This is quite
  1227. # easy to do - just change the next line temporarily to pass debug=True as
  1228. # the last parameter and you'll get a printout of the weightings as well
  1229. # as a map to the lines where next_stage() was called. Of course this isn't
  1230. # critical, but it helps to keep the progress reporting accurate.
  1231. self.rqdata.init_progress_reporter = bb.progress.MultiStageProcessProgressReporter(self.cooker.data,
  1232. "Initialising tasks",
  1233. [43, 967, 4, 3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
  1234. if self.rqdata.prepare() == 0:
  1235. self.state = runQueueComplete
  1236. else:
  1237. self.state = runQueueSceneInit
  1238. if self.state is runQueueSceneInit:
  1239. self.rqdata.init_progress_reporter.next_stage()
  1240. # we are ready to run, emit dependency info to any UI or class which
  1241. # needs it
  1242. depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
  1243. self.rqdata.init_progress_reporter.next_stage()
  1244. bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
  1245. if not self.dm_event_handler_registered:
  1246. res = bb.event.register(self.dm_event_handler_name,
  1247. lambda x: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
  1248. ('bb.event.HeartbeatEvent',))
  1249. self.dm_event_handler_registered = True
  1250. dump = self.cooker.configuration.dump_signatures
  1251. if dump:
  1252. self.rqdata.init_progress_reporter.finish()
  1253. if 'printdiff' in dump:
  1254. invalidtasks = self.print_diffscenetasks()
  1255. self.dump_signatures(dump)
  1256. if 'printdiff' in dump:
  1257. self.write_diffscenetasks(invalidtasks)
  1258. self.state = runQueueComplete
  1259. if self.state is runQueueSceneInit:
  1260. self.rqdata.init_progress_reporter.next_stage()
  1261. self.start_worker()
  1262. self.rqdata.init_progress_reporter.next_stage()
  1263. self.rqexe = RunQueueExecute(self)
  1264. # If we don't have any setscene functions, skip execution
  1265. if len(self.rqdata.runq_setscene_tids) == 0:
  1266. logger.info('No setscene tasks')
  1267. for tid in self.rqdata.runtaskentries:
  1268. if len(self.rqdata.runtaskentries[tid].depends) == 0:
  1269. self.rqexe.setbuildable(tid)
  1270. self.rqexe.tasks_notcovered.add(tid)
  1271. self.rqexe.sqdone = True
  1272. logger.info('Executing Tasks')
  1273. self.state = runQueueRunning
  1274. if self.state is runQueueRunning:
  1275. retval = self.rqexe.execute()
  1276. if self.state is runQueueCleanUp:
  1277. retval = self.rqexe.finish()
  1278. build_done = self.state is runQueueComplete or self.state is runQueueFailed
  1279. if build_done and self.dm_event_handler_registered:
  1280. bb.event.remove(self.dm_event_handler_name, None)
  1281. self.dm_event_handler_registered = False
  1282. if build_done and self.rqexe:
  1283. bb.parse.siggen.save_unitaskhashes()
  1284. self.teardown_workers()
  1285. if self.rqexe:
  1286. if self.rqexe.stats.failed:
  1287. logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed)
  1288. else:
  1289. # Let's avoid the word "failed" if nothing actually did
  1290. logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and all succeeded.", self.rqexe.stats.completed, self.rqexe.stats.skipped)
  1291. if self.state is runQueueFailed:
  1292. raise bb.runqueue.TaskFailure(self.rqexe.failed_tids)
  1293. if self.state is runQueueComplete:
  1294. # All done
  1295. return False
  1296. # Loop
  1297. return retval
  1298. def execute_runqueue(self):
  1299. # Catch unexpected exceptions and ensure we exit when an error occurs, not loop.
  1300. try:
  1301. return self._execute_runqueue()
  1302. except bb.runqueue.TaskFailure:
  1303. raise
  1304. except SystemExit:
  1305. raise
  1306. except bb.BBHandledException:
  1307. try:
  1308. self.teardown_workers()
  1309. except:
  1310. pass
  1311. self.state = runQueueComplete
  1312. raise
  1313. except Exception as err:
  1314. logger.exception("An uncaught exception occurred in runqueue")
  1315. try:
  1316. self.teardown_workers()
  1317. except:
  1318. pass
  1319. self.state = runQueueComplete
  1320. raise
  1321. def finish_runqueue(self, now = False):
  1322. if not self.rqexe:
  1323. self.state = runQueueComplete
  1324. return
  1325. if now:
  1326. self.rqexe.finish_now()
  1327. else:
  1328. self.rqexe.finish()
  1329. def rq_dump_sigfn(self, fn, options):
  1330. bb_cache = bb.cache.NoCache(self.cooker.databuilder)
  1331. the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
  1332. siggen = bb.parse.siggen
  1333. dataCaches = self.rqdata.dataCaches
  1334. siggen.dump_sigfn(fn, dataCaches, options)
  1335. def dump_signatures(self, options):
  1336. fns = set()
  1337. bb.note("Reparsing files to collect dependency data")
  1338. for tid in self.rqdata.runtaskentries:
  1339. fn = fn_from_tid(tid)
  1340. fns.add(fn)
  1341. max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
  1342. # We cannot use the real multiprocessing.Pool easily due to some local data
  1343. # that can't be pickled. This is a cheap multi-process solution.
  1344. launched = []
  1345. while fns:
  1346. if len(launched) < max_process:
  1347. p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options))
  1348. p.start()
  1349. launched.append(p)
  1350. for q in launched:
  1351. # The finished processes are joined when calling is_alive()
  1352. if not q.is_alive():
  1353. launched.remove(q)
  1354. for p in launched:
  1355. p.join()
  1356. bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
  1357. return
  1358. def print_diffscenetasks(self):
  1359. noexec = []
  1360. tocheck = set()
  1361. for tid in self.rqdata.runtaskentries:
  1362. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  1363. taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
  1364. if 'noexec' in taskdep and taskname in taskdep['noexec']:
  1365. noexec.append(tid)
  1366. continue
  1367. tocheck.add(tid)
  1368. valid_new = self.validate_hashes(tocheck, self.cooker.data, None, True)
  1369. # Tasks which are both setscene and noexec never care about dependencies
  1370. # We therefore find tasks which are setscene and noexec and mark their
  1371. # unique dependencies as valid.
  1372. for tid in noexec:
  1373. if tid not in self.rqdata.runq_setscene_tids:
  1374. continue
  1375. for dep in self.rqdata.runtaskentries[tid].depends:
  1376. hasnoexecparents = True
  1377. for dep2 in self.rqdata.runtaskentries[dep].revdeps:
  1378. if dep2 in self.rqdata.runq_setscene_tids and dep2 in noexec:
  1379. continue
  1380. hasnoexecparents = False
  1381. break
  1382. if hasnoexecparents:
  1383. valid_new.add(dep)
  1384. invalidtasks = set()
  1385. for tid in self.rqdata.runtaskentries:
  1386. if tid not in valid_new and tid not in noexec:
  1387. invalidtasks.add(tid)
  1388. found = set()
  1389. processed = set()
  1390. for tid in invalidtasks:
  1391. toprocess = set([tid])
  1392. while toprocess:
  1393. next = set()
  1394. for t in toprocess:
  1395. for dep in self.rqdata.runtaskentries[t].depends:
  1396. if dep in invalidtasks:
  1397. found.add(tid)
  1398. if dep not in processed:
  1399. processed.add(dep)
  1400. next.add(dep)
  1401. toprocess = next
  1402. if tid in found:
  1403. toprocess = set()
  1404. tasklist = []
  1405. for tid in invalidtasks.difference(found):
  1406. tasklist.append(tid)
  1407. if tasklist:
  1408. bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist))
  1409. return invalidtasks.difference(found)
  1410. def write_diffscenetasks(self, invalidtasks):
  1411. # Define recursion callback
  1412. def recursecb(key, hash1, hash2):
  1413. hashes = [hash1, hash2]
  1414. hashfiles = bb.siggen.find_siginfo(key, None, hashes, self.cfgData)
  1415. recout = []
  1416. if len(hashfiles) == 2:
  1417. out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
  1418. recout.extend(list(' ' + l for l in out2))
  1419. else:
  1420. recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
  1421. return recout
  1422. for tid in invalidtasks:
  1423. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  1424. pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
  1425. h = self.rqdata.runtaskentries[tid].hash
  1426. matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData)
  1427. match = None
  1428. for m in matches:
  1429. if h in m:
  1430. match = m
  1431. if match is None:
  1432. bb.fatal("Can't find a task we're supposed to have written out? (hash: %s)?" % h)
  1433. matches = {k : v for k, v in iter(matches.items()) if h not in k}
  1434. if matches:
  1435. latestmatch = sorted(matches.keys(), key=lambda f: matches[f])[-1]
  1436. prevh = __find_sha256__.search(latestmatch).group(0)
  1437. output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb)
  1438. bb.plain("\nTask %s:%s couldn't be used from the cache because:\n We need hash %s, closest matching task was %s\n " % (pn, taskname, h, prevh) + '\n '.join(output))
  1439. class RunQueueExecute:
  1440. def __init__(self, rq):
  1441. self.rq = rq
  1442. self.cooker = rq.cooker
  1443. self.cfgData = rq.cfgData
  1444. self.rqdata = rq.rqdata
  1445. self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS") or 1)
  1446. self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed"
  1447. self.sq_buildable = set()
  1448. self.sq_running = set()
  1449. self.sq_live = set()
  1450. self.updated_taskhash_queue = []
  1451. self.pending_migrations = set()
  1452. self.runq_buildable = set()
  1453. self.runq_running = set()
  1454. self.runq_complete = set()
  1455. self.build_stamps = {}
  1456. self.build_stamps2 = []
  1457. self.failed_tids = []
  1458. self.sq_deferred = {}
  1459. self.stampcache = {}
  1460. self.holdoff_tasks = set()
  1461. self.holdoff_need_update = True
  1462. self.sqdone = False
  1463. self.stats = RunQueueStats(len(self.rqdata.runtaskentries))
  1464. self.sq_stats = RunQueueStats(len(self.rqdata.runq_setscene_tids))
  1465. for mc in rq.worker:
  1466. rq.worker[mc].pipe.setrunqueueexec(self)
  1467. for mc in rq.fakeworker:
  1468. rq.fakeworker[mc].pipe.setrunqueueexec(self)
  1469. if self.number_tasks <= 0:
  1470. bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks)
  1471. # List of setscene tasks which we've covered
  1472. self.scenequeue_covered = set()
  1473. # List of tasks which are covered (including setscene ones)
  1474. self.tasks_covered = set()
  1475. self.tasks_scenequeue_done = set()
  1476. self.scenequeue_notcovered = set()
  1477. self.tasks_notcovered = set()
  1478. self.scenequeue_notneeded = set()
  1479. # We can't skip specified target tasks which aren't setscene tasks
  1480. self.cantskip = set(self.rqdata.target_tids)
  1481. self.cantskip.difference_update(self.rqdata.runq_setscene_tids)
  1482. self.cantskip.intersection_update(self.rqdata.runtaskentries)
  1483. schedulers = self.get_schedulers()
  1484. for scheduler in schedulers:
  1485. if self.scheduler == scheduler.name:
  1486. self.sched = scheduler(self, self.rqdata)
  1487. logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name)
  1488. break
  1489. else:
  1490. bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" %
  1491. (self.scheduler, ", ".join(obj.name for obj in schedulers)))
  1492. #if len(self.rqdata.runq_setscene_tids) > 0:
  1493. self.sqdata = SQData()
  1494. build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
  1495. def runqueue_process_waitpid(self, task, status):
  1496. # self.build_stamps[pid] may not exist when use shared work directory.
  1497. if task in self.build_stamps:
  1498. self.build_stamps2.remove(self.build_stamps[task])
  1499. del self.build_stamps[task]
  1500. if task in self.sq_live:
  1501. if status != 0:
  1502. self.sq_task_fail(task, status)
  1503. else:
  1504. self.sq_task_complete(task)
  1505. self.sq_live.remove(task)
  1506. else:
  1507. if status != 0:
  1508. self.task_fail(task, status)
  1509. else:
  1510. self.task_complete(task)
  1511. return True
  1512. def finish_now(self):
  1513. for mc in self.rq.worker:
  1514. try:
  1515. self.rq.worker[mc].process.stdin.write(b"<finishnow></finishnow>")
  1516. self.rq.worker[mc].process.stdin.flush()
  1517. except IOError:
  1518. # worker must have died?
  1519. pass
  1520. for mc in self.rq.fakeworker:
  1521. try:
  1522. self.rq.fakeworker[mc].process.stdin.write(b"<finishnow></finishnow>")
  1523. self.rq.fakeworker[mc].process.stdin.flush()
  1524. except IOError:
  1525. # worker must have died?
  1526. pass
  1527. if len(self.failed_tids) != 0:
  1528. self.rq.state = runQueueFailed
  1529. return
  1530. self.rq.state = runQueueComplete
  1531. return
  1532. def finish(self):
  1533. self.rq.state = runQueueCleanUp
  1534. active = self.stats.active + self.sq_stats.active
  1535. if active > 0:
  1536. bb.event.fire(runQueueExitWait(active), self.cfgData)
  1537. self.rq.read_workers()
  1538. return self.rq.active_fds()
  1539. if len(self.failed_tids) != 0:
  1540. self.rq.state = runQueueFailed
  1541. return True
  1542. self.rq.state = runQueueComplete
  1543. return True
  1544. # Used by setscene only
  1545. def check_dependencies(self, task, taskdeps):
  1546. if not self.rq.depvalidate:
  1547. return False
  1548. # Must not edit parent data
  1549. taskdeps = set(taskdeps)
  1550. taskdata = {}
  1551. taskdeps.add(task)
  1552. for dep in taskdeps:
  1553. (mc, fn, taskname, taskfn) = split_tid_mcfn(dep)
  1554. pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
  1555. taskdata[dep] = [pn, taskname, fn]
  1556. call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
  1557. locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.data }
  1558. valid = bb.utils.better_eval(call, locs)
  1559. return valid
  1560. def can_start_task(self):
  1561. active = self.stats.active + self.sq_stats.active
  1562. can_start = active < self.number_tasks
  1563. return can_start
  1564. def get_schedulers(self):
  1565. schedulers = set(obj for obj in globals().values()
  1566. if type(obj) is type and
  1567. issubclass(obj, RunQueueScheduler))
  1568. user_schedulers = self.cfgData.getVar("BB_SCHEDULERS")
  1569. if user_schedulers:
  1570. for sched in user_schedulers.split():
  1571. if not "." in sched:
  1572. bb.note("Ignoring scheduler '%s' from BB_SCHEDULERS: not an import" % sched)
  1573. continue
  1574. modname, name = sched.rsplit(".", 1)
  1575. try:
  1576. module = __import__(modname, fromlist=(name,))
  1577. except ImportError as exc:
  1578. logger.critical("Unable to import scheduler '%s' from '%s': %s" % (name, modname, exc))
  1579. raise SystemExit(1)
  1580. else:
  1581. schedulers.add(getattr(module, name))
  1582. return schedulers
  1583. def setbuildable(self, task):
  1584. self.runq_buildable.add(task)
  1585. self.sched.newbuildable(task)
  1586. def task_completeoutright(self, task):
  1587. """
  1588. Mark a task as completed
  1589. Look at the reverse dependencies and mark any task with
  1590. completed dependencies as buildable
  1591. """
  1592. self.runq_complete.add(task)
  1593. for revdep in self.rqdata.runtaskentries[task].revdeps:
  1594. if revdep in self.runq_running:
  1595. continue
  1596. if revdep in self.runq_buildable:
  1597. continue
  1598. alldeps = True
  1599. for dep in self.rqdata.runtaskentries[revdep].depends:
  1600. if dep not in self.runq_complete:
  1601. alldeps = False
  1602. break
  1603. if alldeps:
  1604. self.setbuildable(revdep)
  1605. logger.debug(1, "Marking task %s as buildable", revdep)
  1606. def task_complete(self, task):
  1607. self.stats.taskCompleted()
  1608. bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
  1609. self.task_completeoutright(task)
  1610. def task_fail(self, task, exitcode):
  1611. """
  1612. Called when a task has failed
  1613. Updates the state engine with the failure
  1614. """
  1615. self.stats.taskFailed()
  1616. self.failed_tids.append(task)
  1617. bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
  1618. if self.rqdata.taskData[''].abort:
  1619. self.rq.state = runQueueCleanUp
  1620. def task_skip(self, task, reason):
  1621. self.runq_running.add(task)
  1622. self.setbuildable(task)
  1623. bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
  1624. self.task_completeoutright(task)
  1625. self.stats.taskSkipped()
  1626. self.stats.taskCompleted()
  1627. def summarise_scenequeue_errors(self):
  1628. err = False
  1629. if not self.sqdone:
  1630. logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
  1631. completeevent = sceneQueueComplete(self.sq_stats, self.rq)
  1632. bb.event.fire(completeevent, self.cfgData)
  1633. if self.sq_deferred:
  1634. logger.error("Scenequeue had deferred entries: %s" % pprint.pformat(self.sq_deferred))
  1635. err = True
  1636. if self.updated_taskhash_queue:
  1637. logger.error("Scenequeue had unprocessed changed taskhash entries: %s" % pprint.pformat(self.updated_taskhash_queue))
  1638. err = True
  1639. if self.holdoff_tasks:
  1640. logger.error("Scenequeue had holdoff tasks: %s" % pprint.pformat(self.holdoff_tasks))
  1641. err = True
  1642. for tid in self.rqdata.runq_setscene_tids:
  1643. if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered:
  1644. err = True
  1645. logger.error("Setscene Task %s was never marked as covered or not covered" % tid)
  1646. if tid not in self.sq_buildable:
  1647. err = True
  1648. logger.error("Setscene Task %s was never marked as buildable" % tid)
  1649. if tid not in self.sq_running:
  1650. err = True
  1651. logger.error("Setscene Task %s was never marked as running" % tid)
  1652. for x in self.rqdata.runtaskentries:
  1653. if x not in self.tasks_covered and x not in self.tasks_notcovered:
  1654. logger.error("Task %s was never moved from the setscene queue" % x)
  1655. err = True
  1656. if x not in self.tasks_scenequeue_done:
  1657. logger.error("Task %s was never processed by the setscene code" % x)
  1658. err = True
  1659. if len(self.rqdata.runtaskentries[x].depends) == 0 and x not in self.runq_buildable:
  1660. logger.error("Task %s was never marked as buildable by the setscene code" % x)
  1661. err = True
  1662. return err
  1663. def execute(self):
  1664. """
  1665. Run the tasks in a queue prepared by prepare_runqueue
  1666. """
  1667. self.rq.read_workers()
  1668. self.process_possible_migrations()
  1669. task = None
  1670. if not self.sqdone and self.can_start_task():
  1671. # Find the next setscene to run
  1672. for nexttask in sorted(self.rqdata.runq_setscene_tids):
  1673. if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
  1674. if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
  1675. if nexttask not in self.rqdata.target_tids:
  1676. logger.debug(2, "Skipping setscene for task %s" % nexttask)
  1677. self.sq_task_skip(nexttask)
  1678. self.scenequeue_notneeded.add(nexttask)
  1679. if nexttask in self.sq_deferred:
  1680. del self.sq_deferred[nexttask]
  1681. return True
  1682. # If covered tasks are running, need to wait for them to complete
  1683. for t in self.sqdata.sq_covered_tasks[nexttask]:
  1684. if t in self.runq_running and t not in self.runq_complete:
  1685. continue
  1686. if nexttask in self.sq_deferred:
  1687. if self.sq_deferred[nexttask] not in self.runq_complete:
  1688. continue
  1689. logger.debug(1, "Task %s no longer deferred" % nexttask)
  1690. del self.sq_deferred[nexttask]
  1691. valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, None, False)
  1692. if not valid:
  1693. logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask)
  1694. self.sq_task_failoutright(nexttask)
  1695. return True
  1696. else:
  1697. self.sqdata.outrightfail.remove(nexttask)
  1698. if nexttask in self.sqdata.outrightfail:
  1699. logger.debug(2, 'No package found, so skipping setscene task %s', nexttask)
  1700. self.sq_task_failoutright(nexttask)
  1701. return True
  1702. if nexttask in self.sqdata.unskippable:
  1703. logger.debug(2, "Setscene task %s is unskippable" % nexttask)
  1704. task = nexttask
  1705. break
  1706. if task is not None:
  1707. (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
  1708. taskname = taskname + "_setscene"
  1709. if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache):
  1710. logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task)
  1711. self.sq_task_failoutright(task)
  1712. return True
  1713. if self.cooker.configuration.force:
  1714. if task in self.rqdata.target_tids:
  1715. self.sq_task_failoutright(task)
  1716. return True
  1717. if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
  1718. logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task)
  1719. self.sq_task_skip(task)
  1720. return True
  1721. if self.cooker.configuration.skipsetscene:
  1722. logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task)
  1723. self.sq_task_failoutright(task)
  1724. return True
  1725. startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq)
  1726. bb.event.fire(startevent, self.cfgData)
  1727. taskdepdata = self.sq_build_taskdepdata(task)
  1728. taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
  1729. taskhash = self.rqdata.get_task_hash(task)
  1730. unihash = self.rqdata.get_task_unihash(task)
  1731. if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
  1732. if not mc in self.rq.fakeworker:
  1733. self.rq.start_fakeworker(self, mc)
  1734. self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
  1735. self.rq.fakeworker[mc].process.stdin.flush()
  1736. else:
  1737. self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
  1738. self.rq.worker[mc].process.stdin.flush()
  1739. self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
  1740. self.build_stamps2.append(self.build_stamps[task])
  1741. self.sq_running.add(task)
  1742. self.sq_live.add(task)
  1743. self.sq_stats.taskActive()
  1744. if self.can_start_task():
  1745. return True
  1746. self.update_holdofftasks()
  1747. if not self.sq_live and not self.sqdone and not self.sq_deferred and not self.updated_taskhash_queue and not self.holdoff_tasks:
  1748. logger.info("Setscene tasks completed")
  1749. err = self.summarise_scenequeue_errors()
  1750. if err:
  1751. self.rq.state = runQueueFailed
  1752. return True
  1753. if self.cooker.configuration.setsceneonly:
  1754. self.rq.state = runQueueComplete
  1755. return True
  1756. self.sqdone = True
  1757. if self.stats.total == 0:
  1758. # nothing to do
  1759. self.rq.state = runQueueComplete
  1760. return True
  1761. if self.cooker.configuration.setsceneonly:
  1762. task = None
  1763. else:
  1764. task = self.sched.next()
  1765. if task is not None:
  1766. (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
  1767. if self.rqdata.setscenewhitelist is not None:
  1768. if self.check_setscenewhitelist(task):
  1769. self.task_fail(task, "setscene whitelist")
  1770. return True
  1771. if task in self.tasks_covered:
  1772. logger.debug(2, "Setscene covered task %s", task)
  1773. self.task_skip(task, "covered")
  1774. return True
  1775. if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
  1776. logger.debug(2, "Stamp current task %s", task)
  1777. self.task_skip(task, "existing")
  1778. return True
  1779. taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
  1780. if 'noexec' in taskdep and taskname in taskdep['noexec']:
  1781. startevent = runQueueTaskStarted(task, self.stats, self.rq,
  1782. noexec=True)
  1783. bb.event.fire(startevent, self.cfgData)
  1784. self.runq_running.add(task)
  1785. self.stats.taskActive()
  1786. if not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
  1787. bb.build.make_stamp(taskname, self.rqdata.dataCaches[mc], taskfn)
  1788. self.task_complete(task)
  1789. return True
  1790. else:
  1791. startevent = runQueueTaskStarted(task, self.stats, self.rq)
  1792. bb.event.fire(startevent, self.cfgData)
  1793. taskdepdata = self.build_taskdepdata(task)
  1794. taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
  1795. taskhash = self.rqdata.get_task_hash(task)
  1796. unihash = self.rqdata.get_task_unihash(task)
  1797. if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
  1798. if not mc in self.rq.fakeworker:
  1799. try:
  1800. self.rq.start_fakeworker(self, mc)
  1801. except OSError as exc:
  1802. logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc)))
  1803. self.rq.state = runQueueFailed
  1804. self.stats.taskFailed()
  1805. return True
  1806. self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
  1807. self.rq.fakeworker[mc].process.stdin.flush()
  1808. else:
  1809. self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
  1810. self.rq.worker[mc].process.stdin.flush()
  1811. self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
  1812. self.build_stamps2.append(self.build_stamps[task])
  1813. self.runq_running.add(task)
  1814. self.stats.taskActive()
  1815. if self.can_start_task():
  1816. return True
  1817. if self.stats.active > 0 or self.sq_stats.active > 0:
  1818. self.rq.read_workers()
  1819. return self.rq.active_fds()
  1820. # No more tasks can be run. If we have deferred setscene tasks we should run them.
  1821. if self.sq_deferred:
  1822. tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0])
  1823. logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid)
  1824. self.sq_task_failoutright(tid)
  1825. return True
  1826. if len(self.failed_tids) != 0:
  1827. self.rq.state = runQueueFailed
  1828. return True
  1829. # Sanity Checks
  1830. err = self.summarise_scenequeue_errors()
  1831. for task in self.rqdata.runtaskentries:
  1832. if task not in self.runq_buildable:
  1833. logger.error("Task %s never buildable!", task)
  1834. err = True
  1835. elif task not in self.runq_running:
  1836. logger.error("Task %s never ran!", task)
  1837. err = True
  1838. elif task not in self.runq_complete:
  1839. logger.error("Task %s never completed!", task)
  1840. err = True
  1841. if err:
  1842. self.rq.state = runQueueFailed
  1843. else:
  1844. self.rq.state = runQueueComplete
  1845. return True
  1846. def filtermcdeps(self, task, mc, deps):
  1847. ret = set()
  1848. for dep in deps:
  1849. thismc = mc_from_tid(dep)
  1850. if thismc != mc:
  1851. continue
  1852. ret.add(dep)
  1853. return ret
  1854. # We filter out multiconfig dependencies from taskdepdata we pass to the tasks
  1855. # as most code can't handle them
  1856. def build_taskdepdata(self, task):
  1857. taskdepdata = {}
  1858. mc = mc_from_tid(task)
  1859. next = self.rqdata.runtaskentries[task].depends.copy()
  1860. next.add(task)
  1861. next = self.filtermcdeps(task, mc, next)
  1862. while next:
  1863. additional = []
  1864. for revdep in next:
  1865. (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
  1866. pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
  1867. deps = self.rqdata.runtaskentries[revdep].depends
  1868. provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
  1869. taskhash = self.rqdata.runtaskentries[revdep].hash
  1870. unihash = self.rqdata.runtaskentries[revdep].unihash
  1871. deps = self.filtermcdeps(task, mc, deps)
  1872. taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
  1873. for revdep2 in deps:
  1874. if revdep2 not in taskdepdata:
  1875. additional.append(revdep2)
  1876. next = additional
  1877. #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
  1878. return taskdepdata
  1879. def update_holdofftasks(self):
  1880. if not self.holdoff_need_update:
  1881. return
  1882. notcovered = set(self.scenequeue_notcovered)
  1883. notcovered |= self.cantskip
  1884. for tid in self.scenequeue_notcovered:
  1885. notcovered |= self.sqdata.sq_covered_tasks[tid]
  1886. notcovered |= self.sqdata.unskippable.difference(self.rqdata.runq_setscene_tids)
  1887. notcovered.intersection_update(self.tasks_scenequeue_done)
  1888. covered = set(self.scenequeue_covered)
  1889. for tid in self.scenequeue_covered:
  1890. covered |= self.sqdata.sq_covered_tasks[tid]
  1891. covered.difference_update(notcovered)
  1892. covered.intersection_update(self.tasks_scenequeue_done)
  1893. for tid in notcovered | covered:
  1894. if len(self.rqdata.runtaskentries[tid].depends) == 0:
  1895. self.setbuildable(tid)
  1896. elif self.rqdata.runtaskentries[tid].depends.issubset(self.runq_complete):
  1897. self.setbuildable(tid)
  1898. self.tasks_covered = covered
  1899. self.tasks_notcovered = notcovered
  1900. self.holdoff_tasks = set()
  1901. for tid in self.rqdata.runq_setscene_tids:
  1902. if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered:
  1903. self.holdoff_tasks.add(tid)
  1904. for tid in self.holdoff_tasks.copy():
  1905. for dep in self.sqdata.sq_covered_tasks[tid]:
  1906. if dep not in self.runq_complete:
  1907. self.holdoff_tasks.add(dep)
  1908. self.holdoff_need_update = False
  1909. def process_possible_migrations(self):
  1910. changed = set()
  1911. for tid, unihash in self.updated_taskhash_queue.copy():
  1912. if tid in self.runq_running and tid not in self.runq_complete:
  1913. continue
  1914. self.updated_taskhash_queue.remove((tid, unihash))
  1915. if unihash != self.rqdata.runtaskentries[tid].unihash:
  1916. logger.info("Task %s unihash changed to %s" % (tid, unihash))
  1917. self.rqdata.runtaskentries[tid].unihash = unihash
  1918. bb.parse.siggen.set_unihash(tid, unihash)
  1919. # Work out all tasks which depend on this one
  1920. total = set()
  1921. next = set(self.rqdata.runtaskentries[tid].revdeps)
  1922. while next:
  1923. current = next.copy()
  1924. total = total |next
  1925. next = set()
  1926. for ntid in current:
  1927. next |= self.rqdata.runtaskentries[ntid].revdeps
  1928. next.difference_update(total)
  1929. # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
  1930. done = set()
  1931. next = set(self.rqdata.runtaskentries[tid].revdeps)
  1932. while next:
  1933. current = next.copy()
  1934. next = set()
  1935. for tid in current:
  1936. if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
  1937. continue
  1938. procdep = []
  1939. for dep in self.rqdata.runtaskentries[tid].depends:
  1940. procdep.append(dep)
  1941. orighash = self.rqdata.runtaskentries[tid].hash
  1942. self.rqdata.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
  1943. origuni = self.rqdata.runtaskentries[tid].unihash
  1944. self.rqdata.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
  1945. logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, self.rqdata.runtaskentries[tid].hash, origuni, self.rqdata.runtaskentries[tid].unihash))
  1946. next |= self.rqdata.runtaskentries[tid].revdeps
  1947. changed.add(tid)
  1948. total.remove(tid)
  1949. next.intersection_update(total)
  1950. if changed:
  1951. for mc in self.rq.worker:
  1952. self.rq.worker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
  1953. for mc in self.rq.fakeworker:
  1954. self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
  1955. logger.debug(1, pprint.pformat("Tasks changed:\n%s" % (changed)))
  1956. for tid in changed:
  1957. if tid not in self.rqdata.runq_setscene_tids:
  1958. continue
  1959. valid = self.rq.validate_hashes(set([tid]), self.cooker.data, None, False)
  1960. if not valid:
  1961. continue
  1962. if tid in self.runq_running:
  1963. continue
  1964. if tid not in self.pending_migrations:
  1965. self.pending_migrations.add(tid)
  1966. for tid in self.pending_migrations.copy():
  1967. valid = True
  1968. # Check no tasks this covers are running
  1969. for dep in self.sqdata.sq_covered_tasks[tid]:
  1970. if dep in self.runq_running and dep not in self.runq_complete:
  1971. logger.debug(2, "Task %s is running which blocks setscene for %s from running" % (dep, tid))
  1972. valid = False
  1973. break
  1974. if not valid:
  1975. continue
  1976. self.pending_migrations.remove(tid)
  1977. changed = True
  1978. if tid in self.tasks_scenequeue_done:
  1979. self.tasks_scenequeue_done.remove(tid)
  1980. for dep in self.sqdata.sq_covered_tasks[tid]:
  1981. if dep not in self.runq_complete:
  1982. if dep in self.tasks_scenequeue_done and dep not in self.sqdata.unskippable:
  1983. self.tasks_scenequeue_done.remove(dep)
  1984. if tid in self.sq_buildable:
  1985. self.sq_buildable.remove(tid)
  1986. if tid in self.sq_running:
  1987. self.sq_running.remove(tid)
  1988. if self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
  1989. if tid not in self.sq_buildable:
  1990. self.sq_buildable.add(tid)
  1991. if len(self.sqdata.sq_revdeps[tid]) == 0:
  1992. self.sq_buildable.add(tid)
  1993. if tid in self.sqdata.outrightfail:
  1994. self.sqdata.outrightfail.remove(tid)
  1995. if tid in self.scenequeue_notcovered:
  1996. self.scenequeue_notcovered.remove(tid)
  1997. if tid in self.scenequeue_covered:
  1998. self.scenequeue_covered.remove(tid)
  1999. if tid in self.scenequeue_notneeded:
  2000. self.scenequeue_notneeded.remove(tid)
  2001. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  2002. self.sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", self.rqdata.dataCaches[mc], taskfn, noextra=True)
  2003. if tid in self.stampcache:
  2004. del self.stampcache[tid]
  2005. if tid in self.build_stamps:
  2006. del self.build_stamps[tid]
  2007. logger.info("Setscene task %s now valid and being rerun" % tid)
  2008. self.sqdone = False
  2009. if changed:
  2010. self.holdoff_need_update = True
  2011. def scenequeue_updatecounters(self, task, fail=False):
  2012. for dep in sorted(self.sqdata.sq_deps[task]):
  2013. if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]:
  2014. logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
  2015. self.sq_task_failoutright(dep)
  2016. continue
  2017. if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
  2018. if dep not in self.sq_buildable:
  2019. self.sq_buildable.add(dep)
  2020. next = set([task])
  2021. while next:
  2022. new = set()
  2023. for t in sorted(next):
  2024. self.tasks_scenequeue_done.add(t)
  2025. # Look down the dependency chain for non-setscene things which this task depends on
  2026. # and mark as 'done'
  2027. for dep in self.rqdata.runtaskentries[t].depends:
  2028. if dep in self.rqdata.runq_setscene_tids or dep in self.tasks_scenequeue_done:
  2029. continue
  2030. if self.rqdata.runtaskentries[dep].revdeps.issubset(self.tasks_scenequeue_done):
  2031. new.add(dep)
  2032. next = new
  2033. self.holdoff_need_update = True
  2034. def sq_task_completeoutright(self, task):
  2035. """
  2036. Mark a task as completed
  2037. Look at the reverse dependencies and mark any task with
  2038. completed dependencies as buildable
  2039. """
  2040. logger.debug(1, 'Found task %s which could be accelerated', task)
  2041. self.scenequeue_covered.add(task)
  2042. self.scenequeue_updatecounters(task)
  2043. def sq_check_taskfail(self, task):
  2044. if self.rqdata.setscenewhitelist is not None:
  2045. realtask = task.split('_setscene')[0]
  2046. (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask)
  2047. pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
  2048. if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
  2049. logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
  2050. self.rq.state = runQueueCleanUp
  2051. def sq_task_complete(self, task):
  2052. self.sq_stats.taskCompleted()
  2053. bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData)
  2054. self.sq_task_completeoutright(task)
  2055. def sq_task_fail(self, task, result):
  2056. self.sq_stats.taskFailed()
  2057. bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData)
  2058. self.scenequeue_notcovered.add(task)
  2059. self.scenequeue_updatecounters(task, True)
  2060. self.sq_check_taskfail(task)
  2061. def sq_task_failoutright(self, task):
  2062. self.sq_running.add(task)
  2063. self.sq_buildable.add(task)
  2064. self.sq_stats.taskSkipped()
  2065. self.sq_stats.taskCompleted()
  2066. self.scenequeue_notcovered.add(task)
  2067. self.scenequeue_updatecounters(task, True)
  2068. def sq_task_skip(self, task):
  2069. self.sq_running.add(task)
  2070. self.sq_buildable.add(task)
  2071. self.sq_task_completeoutright(task)
  2072. self.sq_stats.taskSkipped()
  2073. self.sq_stats.taskCompleted()
  2074. def sq_build_taskdepdata(self, task):
  2075. def getsetscenedeps(tid):
  2076. deps = set()
  2077. (mc, fn, taskname, _) = split_tid_mcfn(tid)
  2078. realtid = tid + "_setscene"
  2079. idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends
  2080. for (depname, idependtask) in idepends:
  2081. if depname not in self.rqdata.taskData[mc].build_targets:
  2082. continue
  2083. depfn = self.rqdata.taskData[mc].build_targets[depname][0]
  2084. if depfn is None:
  2085. continue
  2086. deptid = depfn + ":" + idependtask.replace("_setscene", "")
  2087. deps.add(deptid)
  2088. return deps
  2089. taskdepdata = {}
  2090. next = getsetscenedeps(task)
  2091. next.add(task)
  2092. while next:
  2093. additional = []
  2094. for revdep in next:
  2095. (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
  2096. pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
  2097. deps = getsetscenedeps(revdep)
  2098. provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
  2099. taskhash = self.rqdata.runtaskentries[revdep].hash
  2100. unihash = self.rqdata.runtaskentries[revdep].unihash
  2101. taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
  2102. for revdep2 in deps:
  2103. if revdep2 not in taskdepdata:
  2104. additional.append(revdep2)
  2105. next = additional
  2106. #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
  2107. return taskdepdata
  2108. def check_setscenewhitelist(self, tid):
  2109. # Check task that is going to run against the whitelist
  2110. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  2111. # Ignore covered tasks
  2112. if tid in self.tasks_covered:
  2113. return False
  2114. # Ignore stamped tasks
  2115. if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
  2116. return False
  2117. # Ignore noexec tasks
  2118. taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
  2119. if 'noexec' in taskdep and taskname in taskdep['noexec']:
  2120. return False
  2121. pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
  2122. if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
  2123. if tid in self.rqdata.runq_setscene_tids:
  2124. msg = 'Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)
  2125. else:
  2126. msg = 'Task %s.%s attempted to execute unexpectedly' % (pn, taskname)
  2127. logger.error(msg + '\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
  2128. return True
  2129. return False
  2130. class SQData(object):
  2131. def __init__(self):
  2132. # SceneQueue dependencies
  2133. self.sq_deps = {}
  2134. # SceneQueue reverse dependencies
  2135. self.sq_revdeps = {}
  2136. # Injected inter-setscene task dependencies
  2137. self.sq_harddeps = {}
  2138. # Cache of stamp files so duplicates can't run in parallel
  2139. self.stamps = {}
  2140. # Setscene tasks directly depended upon by the build
  2141. self.unskippable = set()
  2142. # List of setscene tasks which aren't present
  2143. self.outrightfail = set()
  2144. # A list of normal tasks a setscene task covers
  2145. self.sq_covered_tasks = {}
  2146. def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
  2147. sq_revdeps = {}
  2148. sq_revdeps_squash = {}
  2149. sq_collated_deps = {}
  2150. # We need to construct a dependency graph for the setscene functions. Intermediate
  2151. # dependencies between the setscene tasks only complicate the code. This code
  2152. # therefore aims to collapse the huge runqueue dependency tree into a smaller one
  2153. # only containing the setscene functions.
  2154. rqdata.init_progress_reporter.next_stage()
  2155. # First process the chains up to the first setscene task.
  2156. endpoints = {}
  2157. for tid in rqdata.runtaskentries:
  2158. sq_revdeps[tid] = copy.copy(rqdata.runtaskentries[tid].revdeps)
  2159. sq_revdeps_squash[tid] = set()
  2160. if (len(sq_revdeps[tid]) == 0) and tid not in rqdata.runq_setscene_tids:
  2161. #bb.warn("Added endpoint %s" % (tid))
  2162. endpoints[tid] = set()
  2163. rqdata.init_progress_reporter.next_stage()
  2164. # Secondly process the chains between setscene tasks.
  2165. for tid in rqdata.runq_setscene_tids:
  2166. sq_collated_deps[tid] = set()
  2167. #bb.warn("Added endpoint 2 %s" % (tid))
  2168. for dep in rqdata.runtaskentries[tid].depends:
  2169. if tid in sq_revdeps[dep]:
  2170. sq_revdeps[dep].remove(tid)
  2171. if dep not in endpoints:
  2172. endpoints[dep] = set()
  2173. #bb.warn(" Added endpoint 3 %s" % (dep))
  2174. endpoints[dep].add(tid)
  2175. rqdata.init_progress_reporter.next_stage()
  2176. def process_endpoints(endpoints):
  2177. newendpoints = {}
  2178. for point, task in endpoints.items():
  2179. tasks = set()
  2180. if task:
  2181. tasks |= task
  2182. if sq_revdeps_squash[point]:
  2183. tasks |= sq_revdeps_squash[point]
  2184. if point not in rqdata.runq_setscene_tids:
  2185. for t in tasks:
  2186. sq_collated_deps[t].add(point)
  2187. sq_revdeps_squash[point] = set()
  2188. if point in rqdata.runq_setscene_tids:
  2189. sq_revdeps_squash[point] = tasks
  2190. tasks = set()
  2191. continue
  2192. for dep in rqdata.runtaskentries[point].depends:
  2193. if point in sq_revdeps[dep]:
  2194. sq_revdeps[dep].remove(point)
  2195. if tasks:
  2196. sq_revdeps_squash[dep] |= tasks
  2197. if len(sq_revdeps[dep]) == 0 and dep not in rqdata.runq_setscene_tids:
  2198. newendpoints[dep] = task
  2199. if len(newendpoints) != 0:
  2200. process_endpoints(newendpoints)
  2201. process_endpoints(endpoints)
  2202. rqdata.init_progress_reporter.next_stage()
  2203. # Build a list of tasks which are "unskippable"
  2204. # These are direct endpoints referenced by the build upto and including setscene tasks
  2205. # Take the build endpoints (no revdeps) and find the sstate tasks they depend upon
  2206. new = True
  2207. for tid in rqdata.runtaskentries:
  2208. if len(rqdata.runtaskentries[tid].revdeps) == 0:
  2209. sqdata.unskippable.add(tid)
  2210. sqdata.unskippable |= sqrq.cantskip
  2211. while new:
  2212. new = False
  2213. orig = sqdata.unskippable.copy()
  2214. for tid in sorted(orig, reverse=True):
  2215. if tid in rqdata.runq_setscene_tids:
  2216. continue
  2217. if len(rqdata.runtaskentries[tid].depends) == 0:
  2218. # These are tasks which have no setscene tasks in their chain, need to mark as directly buildable
  2219. sqrq.setbuildable(tid)
  2220. sqdata.unskippable |= rqdata.runtaskentries[tid].depends
  2221. if sqdata.unskippable != orig:
  2222. new = True
  2223. sqrq.tasks_scenequeue_done |= sqdata.unskippable.difference(rqdata.runq_setscene_tids)
  2224. rqdata.init_progress_reporter.next_stage(len(rqdata.runtaskentries))
  2225. # Sanity check all dependencies could be changed to setscene task references
  2226. for taskcounter, tid in enumerate(rqdata.runtaskentries):
  2227. if tid in rqdata.runq_setscene_tids:
  2228. pass
  2229. elif len(sq_revdeps_squash[tid]) != 0:
  2230. bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
  2231. else:
  2232. del sq_revdeps_squash[tid]
  2233. rqdata.init_progress_reporter.update(taskcounter)
  2234. rqdata.init_progress_reporter.next_stage()
  2235. # Resolve setscene inter-task dependencies
  2236. # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
  2237. # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies
  2238. for tid in rqdata.runq_setscene_tids:
  2239. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  2240. realtid = tid + "_setscene"
  2241. idepends = rqdata.taskData[mc].taskentries[realtid].idepends
  2242. sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", rqdata.dataCaches[mc], taskfn, noextra=True)
  2243. for (depname, idependtask) in idepends:
  2244. if depname not in rqdata.taskData[mc].build_targets:
  2245. continue
  2246. depfn = rqdata.taskData[mc].build_targets[depname][0]
  2247. if depfn is None:
  2248. continue
  2249. deptid = depfn + ":" + idependtask.replace("_setscene", "")
  2250. if deptid not in rqdata.runtaskentries:
  2251. bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask))
  2252. if not deptid in sqdata.sq_harddeps:
  2253. sqdata.sq_harddeps[deptid] = set()
  2254. sqdata.sq_harddeps[deptid].add(tid)
  2255. sq_revdeps_squash[tid].add(deptid)
  2256. # Have to zero this to avoid circular dependencies
  2257. sq_revdeps_squash[deptid] = set()
  2258. rqdata.init_progress_reporter.next_stage()
  2259. for task in sqdata.sq_harddeps:
  2260. for dep in sqdata.sq_harddeps[task]:
  2261. sq_revdeps_squash[dep].add(task)
  2262. rqdata.init_progress_reporter.next_stage()
  2263. #for tid in sq_revdeps_squash:
  2264. # data = ""
  2265. # for dep in sq_revdeps_squash[tid]:
  2266. # data = data + "\n %s" % dep
  2267. # bb.warn("Task %s_setscene: is %s " % (tid, data))
  2268. sqdata.sq_revdeps = sq_revdeps_squash
  2269. sqdata.sq_covered_tasks = sq_collated_deps
  2270. # Build reverse version of revdeps to populate deps structure
  2271. for tid in sqdata.sq_revdeps:
  2272. sqdata.sq_deps[tid] = set()
  2273. for tid in sqdata.sq_revdeps:
  2274. for dep in sqdata.sq_revdeps[tid]:
  2275. sqdata.sq_deps[dep].add(tid)
  2276. rqdata.init_progress_reporter.next_stage()
  2277. multiconfigs = set()
  2278. for tid in sqdata.sq_revdeps:
  2279. multiconfigs.add(mc_from_tid(tid))
  2280. if len(sqdata.sq_revdeps[tid]) == 0:
  2281. sqrq.sq_buildable.add(tid)
  2282. rqdata.init_progress_reporter.finish()
  2283. if rq.hashvalidate:
  2284. noexec = []
  2285. stamppresent = []
  2286. tocheck = set()
  2287. for tid in sorted(sqdata.sq_revdeps):
  2288. (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
  2289. taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
  2290. if 'noexec' in taskdep and taskname in taskdep['noexec']:
  2291. noexec.append(tid)
  2292. sqrq.sq_task_skip(tid)
  2293. bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
  2294. continue
  2295. if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
  2296. logger.debug(2, 'Setscene stamp current for task %s', tid)
  2297. stamppresent.append(tid)
  2298. sqrq.sq_task_skip(tid)
  2299. continue
  2300. if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
  2301. logger.debug(2, 'Normal stamp current for task %s', tid)
  2302. stamppresent.append(tid)
  2303. sqrq.sq_task_skip(tid)
  2304. continue
  2305. tocheck.add(tid)
  2306. valid = rq.validate_hashes(tocheck, cooker.data, len(stamppresent), False)
  2307. valid_new = stamppresent
  2308. for v in valid:
  2309. valid_new.append(v)
  2310. hashes = {}
  2311. for mc in sorted(multiconfigs):
  2312. for tid in sorted(sqdata.sq_revdeps):
  2313. if mc_from_tid(tid) != mc:
  2314. continue
  2315. if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered:
  2316. sqdata.outrightfail.add(tid)
  2317. h = pending_hash_index(tid, rqdata)
  2318. if h not in hashes:
  2319. hashes[h] = tid
  2320. else:
  2321. sqrq.sq_deferred[tid] = hashes[h]
  2322. bb.warn("Deferring %s after %s" % (tid, hashes[h]))
  2323. class TaskFailure(Exception):
  2324. """
  2325. Exception raised when a task in a runqueue fails
  2326. """
  2327. def __init__(self, x):
  2328. self.args = x
  2329. class runQueueExitWait(bb.event.Event):
  2330. """
  2331. Event when waiting for task processes to exit
  2332. """
  2333. def __init__(self, remain):
  2334. self.remain = remain
  2335. self.message = "Waiting for %s active tasks to finish" % remain
  2336. bb.event.Event.__init__(self)
  2337. class runQueueEvent(bb.event.Event):
  2338. """
  2339. Base runQueue event class
  2340. """
  2341. def __init__(self, task, stats, rq):
  2342. self.taskid = task
  2343. self.taskstring = task
  2344. self.taskname = taskname_from_tid(task)
  2345. self.taskfile = fn_from_tid(task)
  2346. self.taskhash = rq.rqdata.get_task_hash(task)
  2347. self.stats = stats.copy()
  2348. bb.event.Event.__init__(self)
  2349. class sceneQueueEvent(runQueueEvent):
  2350. """
  2351. Base sceneQueue event class
  2352. """
  2353. def __init__(self, task, stats, rq, noexec=False):
  2354. runQueueEvent.__init__(self, task, stats, rq)
  2355. self.taskstring = task + "_setscene"
  2356. self.taskname = taskname_from_tid(task) + "_setscene"
  2357. self.taskfile = fn_from_tid(task)
  2358. self.taskhash = rq.rqdata.get_task_hash(task)
  2359. class runQueueTaskStarted(runQueueEvent):
  2360. """
  2361. Event notifying a task was started
  2362. """
  2363. def __init__(self, task, stats, rq, noexec=False):
  2364. runQueueEvent.__init__(self, task, stats, rq)
  2365. self.noexec = noexec
  2366. class sceneQueueTaskStarted(sceneQueueEvent):
  2367. """
  2368. Event notifying a setscene task was started
  2369. """
  2370. def __init__(self, task, stats, rq, noexec=False):
  2371. sceneQueueEvent.__init__(self, task, stats, rq)
  2372. self.noexec = noexec
  2373. class runQueueTaskFailed(runQueueEvent):
  2374. """
  2375. Event notifying a task failed
  2376. """
  2377. def __init__(self, task, stats, exitcode, rq):
  2378. runQueueEvent.__init__(self, task, stats, rq)
  2379. self.exitcode = exitcode
  2380. def __str__(self):
  2381. return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
  2382. class sceneQueueTaskFailed(sceneQueueEvent):
  2383. """
  2384. Event notifying a setscene task failed
  2385. """
  2386. def __init__(self, task, stats, exitcode, rq):
  2387. sceneQueueEvent.__init__(self, task, stats, rq)
  2388. self.exitcode = exitcode
  2389. def __str__(self):
  2390. return "Setscene task (%s) failed with exit code '%s' - real task will be run instead" % (self.taskstring, self.exitcode)
  2391. class sceneQueueComplete(sceneQueueEvent):
  2392. """
  2393. Event when all the sceneQueue tasks are complete
  2394. """
  2395. def __init__(self, stats, rq):
  2396. self.stats = stats.copy()
  2397. bb.event.Event.__init__(self)
  2398. class runQueueTaskCompleted(runQueueEvent):
  2399. """
  2400. Event notifying a task completed
  2401. """
  2402. class sceneQueueTaskCompleted(sceneQueueEvent):
  2403. """
  2404. Event notifying a setscene task completed
  2405. """
  2406. class runQueueTaskSkipped(runQueueEvent):
  2407. """
  2408. Event notifying a task was skipped
  2409. """
  2410. def __init__(self, task, stats, rq, reason):
  2411. runQueueEvent.__init__(self, task, stats, rq)
  2412. self.reason = reason
  2413. class taskUniHashUpdate(bb.event.Event):
  2414. """
  2415. Base runQueue event class
  2416. """
  2417. def __init__(self, task, unihash):
  2418. self.taskid = task
  2419. self.unihash = unihash
  2420. bb.event.Event.__init__(self)
  2421. class runQueuePipe():
  2422. """
  2423. Abstraction for a pipe between a worker thread and the server
  2424. """
  2425. def __init__(self, pipein, pipeout, d, rq, rqexec):
  2426. self.input = pipein
  2427. if pipeout:
  2428. pipeout.close()
  2429. bb.utils.nonblockingfd(self.input)
  2430. self.queue = b""
  2431. self.d = d
  2432. self.rq = rq
  2433. self.rqexec = rqexec
  2434. def setrunqueueexec(self, rqexec):
  2435. self.rqexec = rqexec
  2436. def read(self):
  2437. for workers, name in [(self.rq.worker, "Worker"), (self.rq.fakeworker, "Fakeroot")]:
  2438. for worker in workers.values():
  2439. worker.process.poll()
  2440. if worker.process.returncode is not None and not self.rq.teardown:
  2441. bb.error("%s process (%s) exited unexpectedly (%s), shutting down..." % (name, worker.process.pid, str(worker.process.returncode)))
  2442. self.rq.finish_runqueue(True)
  2443. start = len(self.queue)
  2444. try:
  2445. self.queue = self.queue + (self.input.read(102400) or b"")
  2446. except (OSError, IOError) as e:
  2447. if e.errno != errno.EAGAIN:
  2448. raise
  2449. end = len(self.queue)
  2450. found = True
  2451. while found and len(self.queue):
  2452. found = False
  2453. index = self.queue.find(b"</event>")
  2454. while index != -1 and self.queue.startswith(b"<event>"):
  2455. try:
  2456. event = pickle.loads(self.queue[7:index])
  2457. except ValueError as e:
  2458. bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[7:index]))
  2459. bb.event.fire_from_worker(event, self.d)
  2460. if isinstance(event, taskUniHashUpdate):
  2461. self.rqexec.updated_taskhash_queue.append((event.taskid, event.unihash))
  2462. found = True
  2463. self.queue = self.queue[index+8:]
  2464. index = self.queue.find(b"</event>")
  2465. index = self.queue.find(b"</exitcode>")
  2466. while index != -1 and self.queue.startswith(b"<exitcode>"):
  2467. try:
  2468. task, status = pickle.loads(self.queue[10:index])
  2469. except ValueError as e:
  2470. bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index]))
  2471. self.rqexec.runqueue_process_waitpid(task, status)
  2472. found = True
  2473. self.queue = self.queue[index+11:]
  2474. index = self.queue.find(b"</exitcode>")
  2475. return (end > start)
  2476. def close(self):
  2477. while self.read():
  2478. continue
  2479. if len(self.queue) > 0:
  2480. print("Warning, worker left partial message: %s" % self.queue)
  2481. self.input.close()
  2482. def get_setscene_enforce_whitelist(d):
  2483. if d.getVar('BB_SETSCENE_ENFORCE') != '1':
  2484. return None
  2485. whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST") or "").split()
  2486. outlist = []
  2487. for item in whitelist[:]:
  2488. if item.startswith('%:'):
  2489. for target in sys.argv[1:]:
  2490. if not target.startswith('-'):
  2491. outlist.append(target.split(':')[0] + ':' + item.split(':')[1])
  2492. else:
  2493. outlist.append(item)
  2494. return outlist
  2495. def check_setscene_enforce_whitelist(pn, taskname, whitelist):
  2496. import fnmatch
  2497. if whitelist is not None:
  2498. item = '%s:%s' % (pn, taskname)
  2499. for whitelist_item in whitelist:
  2500. if fnmatch.fnmatch(item, whitelist_item):
  2501. return True
  2502. return False
  2503. return True