process.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656
  1. #
  2. # BitBake Process based server.
  3. #
  4. # Copyright (C) 2010 Bob Foerster <robert@erafx.com>
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. """
  9. This module implements a multiprocessing.Process based server for bitbake.
  10. """
  11. import bb
  12. import bb.event
  13. import logging
  14. import multiprocessing
  15. import threading
  16. import array
  17. import os
  18. import sys
  19. import time
  20. import select
  21. import socket
  22. import subprocess
  23. import errno
  24. import re
  25. import datetime
  26. import bb.server.xmlrpcserver
  27. from bb import daemonize
  28. from multiprocessing import queues
  29. logger = logging.getLogger('BitBake')
  30. class ProcessTimeout(SystemExit):
  31. pass
  32. class ProcessServer(multiprocessing.Process):
  33. profile_filename = "profile.log"
  34. profile_processed_filename = "profile.log.processed"
  35. def __init__(self, lock, sock, sockname):
  36. multiprocessing.Process.__init__(self)
  37. self.command_channel = False
  38. self.command_channel_reply = False
  39. self.quit = False
  40. self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
  41. self.next_heartbeat = time.time()
  42. self.event_handle = None
  43. self.haveui = False
  44. self.lastui = False
  45. self.xmlrpc = False
  46. self._idlefuns = {}
  47. self.bitbake_lock = lock
  48. self.sock = sock
  49. self.sockname = sockname
  50. def register_idle_function(self, function, data):
  51. """Register a function to be called while the server is idle"""
  52. assert hasattr(function, '__call__')
  53. self._idlefuns[function] = data
  54. def run(self):
  55. if self.xmlrpcinterface[0]:
  56. self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self)
  57. print("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port))
  58. heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
  59. if heartbeat_event:
  60. try:
  61. self.heartbeat_seconds = float(heartbeat_event)
  62. except:
  63. bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
  64. self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT')
  65. try:
  66. if self.timeout:
  67. self.timeout = float(self.timeout)
  68. except:
  69. bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
  70. try:
  71. self.bitbake_lock.seek(0)
  72. self.bitbake_lock.truncate()
  73. if self.xmlrpc:
  74. self.bitbake_lock.write("%s %s:%s\n" % (os.getpid(), self.xmlrpc.host, self.xmlrpc.port))
  75. else:
  76. self.bitbake_lock.write("%s\n" % (os.getpid()))
  77. self.bitbake_lock.flush()
  78. except Exception as e:
  79. print("Error writing to lock file: %s" % str(e))
  80. pass
  81. if self.cooker.configuration.profile:
  82. try:
  83. import cProfile as profile
  84. except:
  85. import profile
  86. prof = profile.Profile()
  87. ret = profile.Profile.runcall(prof, self.main)
  88. prof.dump_stats("profile.log")
  89. bb.utils.process_profilelog("profile.log")
  90. print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
  91. else:
  92. ret = self.main()
  93. return ret
  94. def main(self):
  95. self.cooker.pre_serve()
  96. bb.utils.set_process_name("Cooker")
  97. ready = []
  98. newconnections = []
  99. self.controllersock = False
  100. fds = [self.sock]
  101. if self.xmlrpc:
  102. fds.append(self.xmlrpc)
  103. print("Entering server connection loop")
  104. def disconnect_client(self, fds):
  105. print("Disconnecting Client")
  106. if self.controllersock:
  107. fds.remove(self.controllersock)
  108. self.controllersock.close()
  109. self.controllersock = False
  110. if self.haveui:
  111. fds.remove(self.command_channel)
  112. bb.event.unregister_UIHhandler(self.event_handle, True)
  113. self.command_channel_reply.writer.close()
  114. self.event_writer.writer.close()
  115. self.command_channel.close()
  116. self.command_channel = False
  117. del self.event_writer
  118. self.lastui = time.time()
  119. self.cooker.clientComplete()
  120. self.haveui = False
  121. ready = select.select(fds,[],[],0)[0]
  122. if newconnections:
  123. print("Starting new client")
  124. conn = newconnections.pop(-1)
  125. fds.append(conn)
  126. self.controllersock = conn
  127. elif self.timeout is None and not ready:
  128. print("No timeout, exiting.")
  129. self.quit = True
  130. while not self.quit:
  131. if self.sock in ready:
  132. while select.select([self.sock],[],[],0)[0]:
  133. controllersock, address = self.sock.accept()
  134. if self.controllersock:
  135. print("Queuing %s (%s)" % (str(ready), str(newconnections)))
  136. newconnections.append(controllersock)
  137. else:
  138. print("Accepting %s (%s)" % (str(ready), str(newconnections)))
  139. self.controllersock = controllersock
  140. fds.append(controllersock)
  141. if self.controllersock in ready:
  142. try:
  143. print("Processing Client")
  144. ui_fds = recvfds(self.controllersock, 3)
  145. print("Connecting Client")
  146. # Where to write events to
  147. writer = ConnectionWriter(ui_fds[0])
  148. self.event_handle = bb.event.register_UIHhandler(writer, True)
  149. self.event_writer = writer
  150. # Where to read commands from
  151. reader = ConnectionReader(ui_fds[1])
  152. fds.append(reader)
  153. self.command_channel = reader
  154. # Where to send command return values to
  155. writer = ConnectionWriter(ui_fds[2])
  156. self.command_channel_reply = writer
  157. self.haveui = True
  158. except (EOFError, OSError):
  159. disconnect_client(self, fds)
  160. if not self.timeout == -1.0 and not self.haveui and self.lastui and self.timeout and \
  161. (self.lastui + self.timeout) < time.time():
  162. print("Server timeout, exiting.")
  163. self.quit = True
  164. if self.command_channel in ready:
  165. try:
  166. command = self.command_channel.get()
  167. except EOFError:
  168. # Client connection shutting down
  169. ready = []
  170. disconnect_client(self, fds)
  171. continue
  172. if command[0] == "terminateServer":
  173. self.quit = True
  174. continue
  175. try:
  176. print("Running command %s" % command)
  177. self.command_channel_reply.send(self.cooker.command.runCommand(command))
  178. except Exception as e:
  179. logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
  180. if self.xmlrpc in ready:
  181. self.xmlrpc.handle_requests()
  182. ready = self.idle_commands(.1, fds)
  183. print("Exiting")
  184. # Remove the socket file so we don't get any more connections to avoid races
  185. os.unlink(self.sockname)
  186. self.sock.close()
  187. try:
  188. self.cooker.shutdown(True)
  189. self.cooker.notifier.stop()
  190. self.cooker.confignotifier.stop()
  191. except:
  192. pass
  193. self.cooker.post_serve()
  194. # Finally release the lockfile but warn about other processes holding it open
  195. lock = self.bitbake_lock
  196. lockfile = lock.name
  197. lock.close()
  198. lock = None
  199. while not lock:
  200. with bb.utils.timeout(3):
  201. lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
  202. if lock:
  203. # We hold the lock so we can remove the file (hide stale pid data)
  204. bb.utils.remove(lockfile)
  205. bb.utils.unlockfile(lock)
  206. return
  207. if not lock:
  208. # Some systems may not have lsof available
  209. procs = None
  210. try:
  211. procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
  212. except OSError as e:
  213. if e.errno != errno.ENOENT:
  214. raise
  215. if procs is None:
  216. # Fall back to fuser if lsof is unavailable
  217. try:
  218. procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
  219. except OSError as e:
  220. if e.errno != errno.ENOENT:
  221. raise
  222. msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
  223. if procs:
  224. msg += ":\n%s" % str(procs)
  225. print(msg)
  226. def idle_commands(self, delay, fds=None):
  227. nextsleep = delay
  228. if not fds:
  229. fds = []
  230. for function, data in list(self._idlefuns.items()):
  231. try:
  232. retval = function(self, data, False)
  233. if retval is False:
  234. del self._idlefuns[function]
  235. nextsleep = None
  236. elif retval is True:
  237. nextsleep = None
  238. elif isinstance(retval, float) and nextsleep:
  239. if (retval < nextsleep):
  240. nextsleep = retval
  241. elif nextsleep is None:
  242. continue
  243. else:
  244. fds = fds + retval
  245. except SystemExit:
  246. raise
  247. except Exception as exc:
  248. if not isinstance(exc, bb.BBHandledException):
  249. logger.exception('Running idle function')
  250. del self._idlefuns[function]
  251. self.quit = True
  252. # Create new heartbeat event?
  253. now = time.time()
  254. if now >= self.next_heartbeat:
  255. # We might have missed heartbeats. Just trigger once in
  256. # that case and continue after the usual delay.
  257. self.next_heartbeat += self.heartbeat_seconds
  258. if self.next_heartbeat <= now:
  259. self.next_heartbeat = now + self.heartbeat_seconds
  260. heartbeat = bb.event.HeartbeatEvent(now)
  261. bb.event.fire(heartbeat, self.cooker.data)
  262. if nextsleep and now + nextsleep > self.next_heartbeat:
  263. # Shorten timeout so that we we wake up in time for
  264. # the heartbeat.
  265. nextsleep = self.next_heartbeat - now
  266. if nextsleep is not None:
  267. if self.xmlrpc:
  268. nextsleep = self.xmlrpc.get_timeout(nextsleep)
  269. try:
  270. return select.select(fds,[],[],nextsleep)[0]
  271. except InterruptedError:
  272. # Ignore EINTR
  273. return []
  274. else:
  275. return select.select(fds,[],[],0)[0]
  276. class ServerCommunicator():
  277. def __init__(self, connection, recv):
  278. self.connection = connection
  279. self.recv = recv
  280. def runCommand(self, command):
  281. self.connection.send(command)
  282. if not self.recv.poll(30):
  283. raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server")
  284. return self.recv.get()
  285. def updateFeatureSet(self, featureset):
  286. _, error = self.runCommand(["setFeatures", featureset])
  287. if error:
  288. logger.error("Unable to set the cooker to the correct featureset: %s" % error)
  289. raise BaseException(error)
  290. def getEventHandle(self):
  291. handle, error = self.runCommand(["getUIHandlerNum"])
  292. if error:
  293. logger.error("Unable to get UI Handler Number: %s" % error)
  294. raise BaseException(error)
  295. return handle
  296. def terminateServer(self):
  297. self.connection.send(['terminateServer'])
  298. return
  299. class BitBakeProcessServerConnection(object):
  300. def __init__(self, ui_channel, recv, eq, sock):
  301. self.connection = ServerCommunicator(ui_channel, recv)
  302. self.events = eq
  303. # Save sock so it doesn't get gc'd for the life of our connection
  304. self.socket_connection = sock
  305. def terminate(self):
  306. self.socket_connection.close()
  307. self.connection.connection.close()
  308. self.connection.recv.close()
  309. return
  310. class BitBakeServer(object):
  311. start_log_format = '--- Starting bitbake server pid %s at %s ---'
  312. start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
  313. def __init__(self, lock, sockname, configuration, featureset):
  314. self.configuration = configuration
  315. self.featureset = featureset
  316. self.sockname = sockname
  317. self.bitbake_lock = lock
  318. self.readypipe, self.readypipein = os.pipe()
  319. # Create server control socket
  320. if os.path.exists(sockname):
  321. os.unlink(sockname)
  322. # Place the log in the builddirectory alongside the lock file
  323. logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
  324. self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
  325. # AF_UNIX has path length issues so chdir here to workaround
  326. cwd = os.getcwd()
  327. try:
  328. os.chdir(os.path.dirname(sockname))
  329. self.sock.bind(os.path.basename(sockname))
  330. finally:
  331. os.chdir(cwd)
  332. self.sock.listen(1)
  333. os.set_inheritable(self.sock.fileno(), True)
  334. startdatetime = datetime.datetime.now()
  335. bb.daemonize.createDaemon(self._startServer, logfile)
  336. self.sock.close()
  337. self.bitbake_lock.close()
  338. os.close(self.readypipein)
  339. ready = ConnectionReader(self.readypipe)
  340. r = ready.poll(5)
  341. if not r:
  342. bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
  343. r = ready.poll(90)
  344. if r:
  345. try:
  346. r = ready.get()
  347. except EOFError:
  348. # Trap the child exitting/closing the pipe and error out
  349. r = None
  350. if not r or r[0] != "r":
  351. ready.close()
  352. bb.error("Unable to start bitbake server (%s)" % str(r))
  353. if os.path.exists(logfile):
  354. logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
  355. started = False
  356. lines = []
  357. lastlines = []
  358. with open(logfile, "r") as f:
  359. for line in f:
  360. if started:
  361. lines.append(line)
  362. else:
  363. lastlines.append(line)
  364. res = logstart_re.match(line.rstrip())
  365. if res:
  366. ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
  367. if ldatetime >= startdatetime:
  368. started = True
  369. lines.append(line)
  370. if len(lastlines) > 60:
  371. lastlines = lastlines[-60:]
  372. if lines:
  373. if len(lines) > 60:
  374. bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
  375. else:
  376. bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
  377. elif lastlines:
  378. bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
  379. else:
  380. bb.error("%s doesn't exist" % logfile)
  381. raise SystemExit(1)
  382. ready.close()
  383. def _startServer(self):
  384. print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
  385. sys.stdout.flush()
  386. server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
  387. self.configuration.setServerRegIdleCallback(server.register_idle_function)
  388. os.close(self.readypipe)
  389. writer = ConnectionWriter(self.readypipein)
  390. self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
  391. writer.send("r")
  392. writer.close()
  393. server.cooker = self.cooker
  394. server.server_timeout = self.configuration.server_timeout
  395. server.xmlrpcinterface = self.configuration.xmlrpcinterface
  396. print("Started bitbake server pid %d" % os.getpid())
  397. sys.stdout.flush()
  398. server.start()
  399. def connectProcessServer(sockname, featureset):
  400. # Connect to socket
  401. sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
  402. # AF_UNIX has path length issues so chdir here to workaround
  403. cwd = os.getcwd()
  404. readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
  405. eq = command_chan_recv = command_chan = None
  406. sock.settimeout(10)
  407. try:
  408. try:
  409. os.chdir(os.path.dirname(sockname))
  410. finished = False
  411. while not finished:
  412. try:
  413. sock.connect(os.path.basename(sockname))
  414. finished = True
  415. except IOError as e:
  416. if e.errno == errno.EWOULDBLOCK:
  417. pass
  418. raise
  419. finally:
  420. os.chdir(cwd)
  421. # Send an fd for the remote to write events to
  422. readfd, writefd = os.pipe()
  423. eq = BBUIEventQueue(readfd)
  424. # Send an fd for the remote to recieve commands from
  425. readfd1, writefd1 = os.pipe()
  426. command_chan = ConnectionWriter(writefd1)
  427. # Send an fd for the remote to write commands results to
  428. readfd2, writefd2 = os.pipe()
  429. command_chan_recv = ConnectionReader(readfd2)
  430. sendfds(sock, [writefd, readfd1, writefd2])
  431. server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq, sock)
  432. # Close the ends of the pipes we won't use
  433. for i in [writefd, readfd1, writefd2]:
  434. os.close(i)
  435. server_connection.connection.updateFeatureSet(featureset)
  436. except (Exception, SystemExit) as e:
  437. if command_chan_recv:
  438. command_chan_recv.close()
  439. if command_chan:
  440. command_chan.close()
  441. for i in [writefd, readfd1, writefd2]:
  442. try:
  443. if i:
  444. os.close(i)
  445. except OSError:
  446. pass
  447. sock.close()
  448. raise
  449. return server_connection
  450. def sendfds(sock, fds):
  451. '''Send an array of fds over an AF_UNIX socket.'''
  452. fds = array.array('i', fds)
  453. msg = bytes([len(fds) % 256])
  454. sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
  455. def recvfds(sock, size):
  456. '''Receive an array of fds over an AF_UNIX socket.'''
  457. a = array.array('i')
  458. bytes_size = a.itemsize * size
  459. msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
  460. if not msg and not ancdata:
  461. raise EOFError
  462. try:
  463. if len(ancdata) != 1:
  464. raise RuntimeError('received %d items of ancdata' %
  465. len(ancdata))
  466. cmsg_level, cmsg_type, cmsg_data = ancdata[0]
  467. if (cmsg_level == socket.SOL_SOCKET and
  468. cmsg_type == socket.SCM_RIGHTS):
  469. if len(cmsg_data) % a.itemsize != 0:
  470. raise ValueError
  471. a.frombytes(cmsg_data)
  472. assert len(a) % 256 == msg[0]
  473. return list(a)
  474. except (ValueError, IndexError):
  475. pass
  476. raise RuntimeError('Invalid data received')
  477. class BBUIEventQueue:
  478. def __init__(self, readfd):
  479. self.eventQueue = []
  480. self.eventQueueLock = threading.Lock()
  481. self.eventQueueNotify = threading.Event()
  482. self.reader = ConnectionReader(readfd)
  483. self.t = threading.Thread()
  484. self.t.setDaemon(True)
  485. self.t.run = self.startCallbackHandler
  486. self.t.start()
  487. def getEvent(self):
  488. self.eventQueueLock.acquire()
  489. if len(self.eventQueue) == 0:
  490. self.eventQueueLock.release()
  491. return None
  492. item = self.eventQueue.pop(0)
  493. if len(self.eventQueue) == 0:
  494. self.eventQueueNotify.clear()
  495. self.eventQueueLock.release()
  496. return item
  497. def waitEvent(self, delay):
  498. self.eventQueueNotify.wait(delay)
  499. return self.getEvent()
  500. def queue_event(self, event):
  501. self.eventQueueLock.acquire()
  502. self.eventQueue.append(event)
  503. self.eventQueueNotify.set()
  504. self.eventQueueLock.release()
  505. def send_event(self, event):
  506. self.queue_event(pickle.loads(event))
  507. def startCallbackHandler(self):
  508. bb.utils.set_process_name("UIEventQueue")
  509. while True:
  510. try:
  511. self.reader.wait()
  512. event = self.reader.get()
  513. self.queue_event(event)
  514. except EOFError:
  515. # Easiest way to exit is to close the file descriptor to cause an exit
  516. break
  517. self.reader.close()
  518. class ConnectionReader(object):
  519. def __init__(self, fd):
  520. self.reader = multiprocessing.connection.Connection(fd, writable=False)
  521. self.rlock = multiprocessing.Lock()
  522. def wait(self, timeout=None):
  523. return multiprocessing.connection.wait([self.reader], timeout)
  524. def poll(self, timeout=None):
  525. return self.reader.poll(timeout)
  526. def get(self):
  527. with self.rlock:
  528. res = self.reader.recv_bytes()
  529. return multiprocessing.reduction.ForkingPickler.loads(res)
  530. def fileno(self):
  531. return self.reader.fileno()
  532. def close(self):
  533. return self.reader.close()
  534. class ConnectionWriter(object):
  535. def __init__(self, fd):
  536. self.writer = multiprocessing.connection.Connection(fd, readable=False)
  537. self.wlock = multiprocessing.Lock()
  538. # Why bb.event needs this I have no idea
  539. self.event = self
  540. def send(self, obj):
  541. obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
  542. with self.wlock:
  543. self.writer.send_bytes(obj)
  544. def fileno(self):
  545. return self.writer.fileno()
  546. def close(self):
  547. return self.writer.close()