monitordisk.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright (C) 2012 Robert Yang
  6. #
  7. # SPDX-License-Identifier: GPL-2.0-only
  8. #
  9. # This program is free software; you can redistribute it and/or modify
  10. # it under the terms of the GNU General Public License version 2 as
  11. # published by the Free Software Foundation.
  12. #
  13. # This program is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU General Public License along
  19. # with this program; if not, write to the Free Software Foundation, Inc.,
  20. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21. import os, logging, re, sys
  22. import bb
  23. logger = logging.getLogger("BitBake.Monitor")
  24. def printErr(info):
  25. logger.error("%s\n Disk space monitor will NOT be enabled" % info)
  26. def convertGMK(unit):
  27. """ Convert the space unit G, M, K, the unit is case-insensitive """
  28. unitG = re.match(r'([1-9][0-9]*)[gG]\s?$', unit)
  29. if unitG:
  30. return int(unitG.group(1)) * (1024 ** 3)
  31. unitM = re.match(r'([1-9][0-9]*)[mM]\s?$', unit)
  32. if unitM:
  33. return int(unitM.group(1)) * (1024 ** 2)
  34. unitK = re.match(r'([1-9][0-9]*)[kK]\s?$', unit)
  35. if unitK:
  36. return int(unitK.group(1)) * 1024
  37. unitN = re.match(r'([1-9][0-9]*)\s?$', unit)
  38. if unitN:
  39. return int(unitN.group(1))
  40. else:
  41. return None
  42. def getMountedDev(path):
  43. """ Get the device mounted at the path, uses /proc/mounts """
  44. # Get the mount point of the filesystem containing path
  45. # st_dev is the ID of device containing file
  46. parentDev = os.stat(path).st_dev
  47. currentDev = parentDev
  48. # When the current directory's device is different from the
  49. # parent's, then the current directory is a mount point
  50. while parentDev == currentDev:
  51. mountPoint = path
  52. # Use dirname to get the parent's directory
  53. path = os.path.dirname(path)
  54. # Reach the "/"
  55. if path == mountPoint:
  56. break
  57. parentDev= os.stat(path).st_dev
  58. try:
  59. with open("/proc/mounts", "r") as ifp:
  60. for line in ifp:
  61. procLines = line.rstrip('\n').split()
  62. if procLines[1] == mountPoint:
  63. return procLines[0]
  64. except EnvironmentError:
  65. pass
  66. return None
  67. def getDiskData(BBDirs, configuration):
  68. """Prepare disk data for disk space monitor"""
  69. # Save the device IDs, need the ID to be unique (the dictionary's key is
  70. # unique), so that when more than one directory is located on the same
  71. # device, we just monitor it once
  72. devDict = {}
  73. for pathSpaceInode in BBDirs.split():
  74. # The input format is: "dir,space,inode", dir is a must, space
  75. # and inode are optional
  76. pathSpaceInodeRe = re.match(r'([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
  77. if not pathSpaceInodeRe:
  78. printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
  79. return None
  80. action = pathSpaceInodeRe.group(1)
  81. if action not in ("ABORT", "STOPTASKS", "WARN"):
  82. printErr("Unknown disk space monitor action: %s" % action)
  83. return None
  84. path = os.path.realpath(pathSpaceInodeRe.group(2))
  85. if not path:
  86. printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
  87. return None
  88. # The disk space or inode is optional, but it should have a correct
  89. # value once it is specified
  90. minSpace = pathSpaceInodeRe.group(3)
  91. if minSpace:
  92. minSpace = convertGMK(minSpace)
  93. if not minSpace:
  94. printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3))
  95. return None
  96. else:
  97. # None means that it is not specified
  98. minSpace = None
  99. minInode = pathSpaceInodeRe.group(4)
  100. if minInode:
  101. minInode = convertGMK(minInode)
  102. if not minInode:
  103. printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4))
  104. return None
  105. else:
  106. # None means that it is not specified
  107. minInode = None
  108. if minSpace is None and minInode is None:
  109. printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode)
  110. return None
  111. # mkdir for the directory since it may not exist, for example the
  112. # DL_DIR may not exist at the very beginning
  113. if not os.path.exists(path):
  114. bb.utils.mkdirhier(path)
  115. dev = getMountedDev(path)
  116. # Use path/action as the key
  117. devDict[(path, action)] = [dev, minSpace, minInode]
  118. return devDict
  119. def getInterval(configuration):
  120. """ Get the disk space interval """
  121. # The default value is 50M and 5K.
  122. spaceDefault = 50 * 1024 * 1024
  123. inodeDefault = 5 * 1024
  124. interval = configuration.getVar("BB_DISKMON_WARNINTERVAL")
  125. if not interval:
  126. return spaceDefault, inodeDefault
  127. else:
  128. # The disk space or inode interval is optional, but it should
  129. # have a correct value once it is specified
  130. intervalRe = re.match(r'([^,]*),?\s*(.*)', interval)
  131. if intervalRe:
  132. intervalSpace = intervalRe.group(1)
  133. if intervalSpace:
  134. intervalSpace = convertGMK(intervalSpace)
  135. if not intervalSpace:
  136. printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1))
  137. return None, None
  138. else:
  139. intervalSpace = spaceDefault
  140. intervalInode = intervalRe.group(2)
  141. if intervalInode:
  142. intervalInode = convertGMK(intervalInode)
  143. if not intervalInode:
  144. printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2))
  145. return None, None
  146. else:
  147. intervalInode = inodeDefault
  148. return intervalSpace, intervalInode
  149. else:
  150. printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval)
  151. return None, None
  152. class diskMonitor:
  153. """Prepare the disk space monitor data"""
  154. def __init__(self, configuration):
  155. self.enableMonitor = False
  156. self.configuration = configuration
  157. BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
  158. if BBDirs:
  159. self.devDict = getDiskData(BBDirs, configuration)
  160. if self.devDict:
  161. self.spaceInterval, self.inodeInterval = getInterval(configuration)
  162. if self.spaceInterval and self.inodeInterval:
  163. self.enableMonitor = True
  164. # These are for saving the previous disk free space and inode, we
  165. # use them to avoid printing too many warning messages
  166. self.preFreeS = {}
  167. self.preFreeI = {}
  168. # This is for STOPTASKS and ABORT, to avoid printing the message
  169. # repeatedly while waiting for the tasks to finish
  170. self.checked = {}
  171. for k in self.devDict:
  172. self.preFreeS[k] = 0
  173. self.preFreeI[k] = 0
  174. self.checked[k] = False
  175. if self.spaceInterval is None and self.inodeInterval is None:
  176. self.enableMonitor = False
  177. def check(self, rq):
  178. """ Take action for the monitor """
  179. if self.enableMonitor:
  180. diskUsage = {}
  181. for k, attributes in self.devDict.items():
  182. path, action = k
  183. dev, minSpace, minInode = attributes
  184. st = os.statvfs(path)
  185. # The available free space, integer number
  186. freeSpace = st.f_bavail * st.f_frsize
  187. # Send all relevant information in the event.
  188. freeSpaceRoot = st.f_bfree * st.f_frsize
  189. totalSpace = st.f_blocks * st.f_frsize
  190. diskUsage[dev] = bb.event.DiskUsageSample(freeSpace, freeSpaceRoot, totalSpace)
  191. if minSpace and freeSpace < minSpace:
  192. # Always show warning, the self.checked would always be False if the action is WARN
  193. if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]:
  194. logger.warning("The free space of %s (%s) is running low (%.3fGB left)" % \
  195. (path, dev, freeSpace / 1024 / 1024 / 1024.0))
  196. self.preFreeS[k] = freeSpace
  197. if action == "STOPTASKS" and not self.checked[k]:
  198. logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
  199. self.checked[k] = True
  200. rq.finish_runqueue(False)
  201. bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
  202. elif action == "ABORT" and not self.checked[k]:
  203. logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
  204. self.checked[k] = True
  205. rq.finish_runqueue(True)
  206. bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
  207. # The free inodes, integer number
  208. freeInode = st.f_favail
  209. if minInode and freeInode < minInode:
  210. # Some filesystems use dynamic inodes so can't run out
  211. # (e.g. btrfs). This is reported by the inode count being 0.
  212. if st.f_files == 0:
  213. self.devDict[k][2] = None
  214. continue
  215. # Always show warning, the self.checked would always be False if the action is WARN
  216. if self.preFreeI[k] == 0 or self.preFreeI[k] - freeInode > self.inodeInterval and not self.checked[k]:
  217. logger.warning("The free inode of %s (%s) is running low (%.3fK left)" % \
  218. (path, dev, freeInode / 1024.0))
  219. self.preFreeI[k] = freeInode
  220. if action == "STOPTASKS" and not self.checked[k]:
  221. logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
  222. self.checked[k] = True
  223. rq.finish_runqueue(False)
  224. bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
  225. elif action == "ABORT" and not self.checked[k]:
  226. logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
  227. self.checked[k] = True
  228. rq.finish_runqueue(True)
  229. bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
  230. bb.event.fire(bb.event.MonitorDiskEvent(diskUsage), self.configuration)
  231. return