monitordisk.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright (C) 2012 Robert Yang
  6. #
  7. # SPDX-License-Identifier: GPL-2.0-only
  8. #
  9. import os, logging, re, sys
  10. import bb
  11. logger = logging.getLogger("BitBake.Monitor")
  12. def printErr(info):
  13. logger.error("%s\n Disk space monitor will NOT be enabled" % info)
  14. def convertGMK(unit):
  15. """ Convert the space unit G, M, K, the unit is case-insensitive """
  16. unitG = re.match(r'([1-9][0-9]*)[gG]\s?$', unit)
  17. if unitG:
  18. return int(unitG.group(1)) * (1024 ** 3)
  19. unitM = re.match(r'([1-9][0-9]*)[mM]\s?$', unit)
  20. if unitM:
  21. return int(unitM.group(1)) * (1024 ** 2)
  22. unitK = re.match(r'([1-9][0-9]*)[kK]\s?$', unit)
  23. if unitK:
  24. return int(unitK.group(1)) * 1024
  25. unitN = re.match(r'([1-9][0-9]*)\s?$', unit)
  26. if unitN:
  27. return int(unitN.group(1))
  28. else:
  29. return None
  30. def getMountedDev(path):
  31. """ Get the device mounted at the path, uses /proc/mounts """
  32. # Get the mount point of the filesystem containing path
  33. # st_dev is the ID of device containing file
  34. parentDev = os.stat(path).st_dev
  35. currentDev = parentDev
  36. # When the current directory's device is different from the
  37. # parent's, then the current directory is a mount point
  38. while parentDev == currentDev:
  39. mountPoint = path
  40. # Use dirname to get the parent's directory
  41. path = os.path.dirname(path)
  42. # Reach the "/"
  43. if path == mountPoint:
  44. break
  45. parentDev= os.stat(path).st_dev
  46. try:
  47. with open("/proc/mounts", "r") as ifp:
  48. for line in ifp:
  49. procLines = line.rstrip('\n').split()
  50. if procLines[1] == mountPoint:
  51. return procLines[0]
  52. except EnvironmentError:
  53. pass
  54. return None
  55. def getDiskData(BBDirs, configuration):
  56. """Prepare disk data for disk space monitor"""
  57. # Save the device IDs, need the ID to be unique (the dictionary's key is
  58. # unique), so that when more than one directory is located on the same
  59. # device, we just monitor it once
  60. devDict = {}
  61. for pathSpaceInode in BBDirs.split():
  62. # The input format is: "dir,space,inode", dir is a must, space
  63. # and inode are optional
  64. pathSpaceInodeRe = re.match(r'([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
  65. if not pathSpaceInodeRe:
  66. printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
  67. return None
  68. action = pathSpaceInodeRe.group(1)
  69. if action not in ("ABORT", "STOPTASKS", "WARN"):
  70. printErr("Unknown disk space monitor action: %s" % action)
  71. return None
  72. path = os.path.realpath(pathSpaceInodeRe.group(2))
  73. if not path:
  74. printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
  75. return None
  76. # The disk space or inode is optional, but it should have a correct
  77. # value once it is specified
  78. minSpace = pathSpaceInodeRe.group(3)
  79. if minSpace:
  80. minSpace = convertGMK(minSpace)
  81. if not minSpace:
  82. printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3))
  83. return None
  84. else:
  85. # None means that it is not specified
  86. minSpace = None
  87. minInode = pathSpaceInodeRe.group(4)
  88. if minInode:
  89. minInode = convertGMK(minInode)
  90. if not minInode:
  91. printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4))
  92. return None
  93. else:
  94. # None means that it is not specified
  95. minInode = None
  96. if minSpace is None and minInode is None:
  97. printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode)
  98. return None
  99. # mkdir for the directory since it may not exist, for example the
  100. # DL_DIR may not exist at the very beginning
  101. if not os.path.exists(path):
  102. bb.utils.mkdirhier(path)
  103. dev = getMountedDev(path)
  104. # Use path/action as the key
  105. devDict[(path, action)] = [dev, minSpace, minInode]
  106. return devDict
  107. def getInterval(configuration):
  108. """ Get the disk space interval """
  109. # The default value is 50M and 5K.
  110. spaceDefault = 50 * 1024 * 1024
  111. inodeDefault = 5 * 1024
  112. interval = configuration.getVar("BB_DISKMON_WARNINTERVAL")
  113. if not interval:
  114. return spaceDefault, inodeDefault
  115. else:
  116. # The disk space or inode interval is optional, but it should
  117. # have a correct value once it is specified
  118. intervalRe = re.match(r'([^,]*),?\s*(.*)', interval)
  119. if intervalRe:
  120. intervalSpace = intervalRe.group(1)
  121. if intervalSpace:
  122. intervalSpace = convertGMK(intervalSpace)
  123. if not intervalSpace:
  124. printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1))
  125. return None, None
  126. else:
  127. intervalSpace = spaceDefault
  128. intervalInode = intervalRe.group(2)
  129. if intervalInode:
  130. intervalInode = convertGMK(intervalInode)
  131. if not intervalInode:
  132. printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2))
  133. return None, None
  134. else:
  135. intervalInode = inodeDefault
  136. return intervalSpace, intervalInode
  137. else:
  138. printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval)
  139. return None, None
  140. class diskMonitor:
  141. """Prepare the disk space monitor data"""
  142. def __init__(self, configuration):
  143. self.enableMonitor = False
  144. self.configuration = configuration
  145. BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
  146. if BBDirs:
  147. self.devDict = getDiskData(BBDirs, configuration)
  148. if self.devDict:
  149. self.spaceInterval, self.inodeInterval = getInterval(configuration)
  150. if self.spaceInterval and self.inodeInterval:
  151. self.enableMonitor = True
  152. # These are for saving the previous disk free space and inode, we
  153. # use them to avoid printing too many warning messages
  154. self.preFreeS = {}
  155. self.preFreeI = {}
  156. # This is for STOPTASKS and ABORT, to avoid printing the message
  157. # repeatedly while waiting for the tasks to finish
  158. self.checked = {}
  159. for k in self.devDict:
  160. self.preFreeS[k] = 0
  161. self.preFreeI[k] = 0
  162. self.checked[k] = False
  163. if self.spaceInterval is None and self.inodeInterval is None:
  164. self.enableMonitor = False
  165. def check(self, rq):
  166. """ Take action for the monitor """
  167. if self.enableMonitor:
  168. diskUsage = {}
  169. for k, attributes in self.devDict.items():
  170. path, action = k
  171. dev, minSpace, minInode = attributes
  172. st = os.statvfs(path)
  173. # The available free space, integer number
  174. freeSpace = st.f_bavail * st.f_frsize
  175. # Send all relevant information in the event.
  176. freeSpaceRoot = st.f_bfree * st.f_frsize
  177. totalSpace = st.f_blocks * st.f_frsize
  178. diskUsage[dev] = bb.event.DiskUsageSample(freeSpace, freeSpaceRoot, totalSpace)
  179. if minSpace and freeSpace < minSpace:
  180. # Always show warning, the self.checked would always be False if the action is WARN
  181. if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]:
  182. logger.warning("The free space of %s (%s) is running low (%.3fGB left)" % \
  183. (path, dev, freeSpace / 1024 / 1024 / 1024.0))
  184. self.preFreeS[k] = freeSpace
  185. if action == "STOPTASKS" and not self.checked[k]:
  186. logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
  187. self.checked[k] = True
  188. rq.finish_runqueue(False)
  189. bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
  190. elif action == "ABORT" and not self.checked[k]:
  191. logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
  192. self.checked[k] = True
  193. rq.finish_runqueue(True)
  194. bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
  195. # The free inodes, integer number
  196. freeInode = st.f_favail
  197. if minInode and freeInode < minInode:
  198. # Some filesystems use dynamic inodes so can't run out
  199. # (e.g. btrfs). This is reported by the inode count being 0.
  200. if st.f_files == 0:
  201. self.devDict[k][2] = None
  202. continue
  203. # Always show warning, the self.checked would always be False if the action is WARN
  204. if self.preFreeI[k] == 0 or self.preFreeI[k] - freeInode > self.inodeInterval and not self.checked[k]:
  205. logger.warning("The free inode of %s (%s) is running low (%.3fK left)" % \
  206. (path, dev, freeInode / 1024.0))
  207. self.preFreeI[k] = freeInode
  208. if action == "STOPTASKS" and not self.checked[k]:
  209. logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
  210. self.checked[k] = True
  211. rq.finish_runqueue(False)
  212. bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
  213. elif action == "ABORT" and not self.checked[k]:
  214. logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
  215. self.checked[k] = True
  216. rq.finish_runqueue(True)
  217. bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
  218. bb.event.fire(bb.event.MonitorDiskEvent(diskUsage), self.configuration)
  219. return