123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161 |
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- # Implements system state sampling. Called by buildstats.bbclass.
- # Because it is a real Python module, it can hold persistent state,
- # like open log files and the time of the last sampling.
- import time
- import re
- import bb.event
- class SystemStats:
- def __init__(self, d):
- bn = d.getVar('BUILDNAME')
- bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn)
- bb.utils.mkdirhier(bsdir)
- self.proc_files = []
- for filename, handler in (
- ('diskstats', self._reduce_diskstats),
- ('meminfo', self._reduce_meminfo),
- ('stat', self._reduce_stat),
- ):
- # The corresponding /proc files might not exist on the host.
- # For example, /proc/diskstats is not available in virtualized
- # environments like Linux-VServer. Silently skip collecting
- # the data.
- if os.path.exists(os.path.join('/proc', filename)):
- # In practice, this class gets instantiated only once in
- # the bitbake cooker process. Therefore 'append' mode is
- # not strictly necessary, but using it makes the class
- # more robust should two processes ever write
- # concurrently.
- destfile = os.path.join(bsdir, '%sproc_%s.log' % ('reduced_' if handler else '', filename))
- self.proc_files.append((filename, open(destfile, 'ab'), handler))
- self.monitor_disk = open(os.path.join(bsdir, 'monitor_disk.log'), 'ab')
- # Last time that we sampled /proc data resp. recorded disk monitoring data.
- self.last_proc = 0
- self.last_disk_monitor = 0
- # Minimum number of seconds between recording a sample. This
- # becames relevant when we get called very often while many
- # short tasks get started. Sampling during quiet periods
- # depends on the heartbeat event, which fires less often.
- self.min_seconds = 1
- self.meminfo_regex = re.compile(b'^(MemTotal|MemFree|Buffers|Cached|SwapTotal|SwapFree):\s*(\d+)')
- self.diskstats_regex = re.compile(b'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$')
- self.diskstats_ltime = None
- self.diskstats_data = None
- self.stat_ltimes = None
- def close(self):
- self.monitor_disk.close()
- for _, output, _ in self.proc_files:
- output.close()
- def _reduce_meminfo(self, time, data):
- """
- Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree'
- and writes their values into a single line, in that order.
- """
- values = {}
- for line in data.split(b'\n'):
- m = self.meminfo_regex.match(line)
- if m:
- values[m.group(1)] = m.group(2)
- if len(values) == 6:
- return (time,
- b' '.join([values[x] for x in
- (b'MemTotal', b'MemFree', b'Buffers', b'Cached', b'SwapTotal', b'SwapFree')]) + b'\n')
- def _diskstats_is_relevant_line(self, linetokens):
- if len(linetokens) != 14:
- return False
- disk = linetokens[2]
- return self.diskstats_regex.match(disk)
- def _reduce_diskstats(self, time, data):
- relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n')))
- diskdata = [0] * 3
- reduced = None
- for tokens in relevant_tokens:
- # rsect
- diskdata[0] += int(tokens[5])
- # wsect
- diskdata[1] += int(tokens[9])
- # use
- diskdata[2] += int(tokens[12])
- if self.diskstats_ltime:
- # We need to compute information about the time interval
- # since the last sampling and record the result as sample
- # for that point in the past.
- interval = time - self.diskstats_ltime
- if interval > 0:
- sums = [ a - b for a, b in zip(diskdata, self.diskstats_data) ]
- readTput = sums[0] / 2.0 * 100.0 / interval
- writeTput = sums[1] / 2.0 * 100.0 / interval
- util = float( sums[2] ) / 10 / interval
- util = max(0.0, min(1.0, util))
- reduced = (self.diskstats_ltime, (readTput, writeTput, util))
- self.diskstats_ltime = time
- self.diskstats_data = diskdata
- return reduced
- def _reduce_nop(self, time, data):
- return (time, data)
- def _reduce_stat(self, time, data):
- if not data:
- return None
- # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line
- tokens = data.split(b'\n', 1)[0].split()
- times = [ int(token) for token in tokens[1:] ]
- reduced = None
- if self.stat_ltimes:
- user = float((times[0] + times[1]) - (self.stat_ltimes[0] + self.stat_ltimes[1]))
- system = float((times[2] + times[5] + times[6]) - (self.stat_ltimes[2] + self.stat_ltimes[5] + self.stat_ltimes[6]))
- idle = float(times[3] - self.stat_ltimes[3])
- iowait = float(times[4] - self.stat_ltimes[4])
- aSum = max(user + system + idle + iowait, 1)
- reduced = (time, (user/aSum, system/aSum, iowait/aSum))
- self.stat_ltimes = times
- return reduced
- def sample(self, event, force):
- now = time.time()
- if (now - self.last_proc > self.min_seconds) or force:
- for filename, output, handler in self.proc_files:
- with open(os.path.join('/proc', filename), 'rb') as input:
- data = input.read()
- if handler:
- reduced = handler(now, data)
- else:
- reduced = (now, data)
- if reduced:
- if isinstance(reduced[1], bytes):
- # Use as it is.
- data = reduced[1]
- else:
- # Convert to a single line.
- data = (' '.join([str(x) for x in reduced[1]]) + '\n').encode('ascii')
- # Unbuffered raw write, less overhead and useful
- # in case that we end up with concurrent writes.
- os.write(output.fileno(),
- ('%.0f\n' % reduced[0]).encode('ascii') +
- data +
- b'\n')
- self.last_proc = now
- if isinstance(event, bb.event.MonitorDiskEvent) and \
- ((now - self.last_disk_monitor > self.min_seconds) or force):
- os.write(self.monitor_disk.fileno(),
- ('%.0f\n' % now).encode('ascii') +
- ''.join(['%s: %d\n' % (dev, sample.total_bytes - sample.free_bytes)
- for dev, sample in event.disk_usage.items()]).encode('ascii') +
- b'\n')
- self.last_disk_monitor = now
|