size-stats 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308
  1. #!/usr/bin/env python
  2. # Copyright (C) 2014 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
  3. # This program is free software; you can redistribute it and/or modify
  4. # it under the terms of the GNU General Public License as published by
  5. # the Free Software Foundation; either version 2 of the License, or
  6. # (at your option) any later version.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. # General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License
  14. # along with this program; if not, write to the Free Software
  15. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  16. import sys
  17. import os
  18. import os.path
  19. import argparse
  20. import csv
  21. import collections
  22. import math
  23. try:
  24. import matplotlib
  25. matplotlib.use('Agg')
  26. import matplotlib.font_manager as fm
  27. import matplotlib.pyplot as plt
  28. except ImportError:
  29. sys.stderr.write("You need python-matplotlib to generate the size graph\n")
  30. exit(1)
  31. class Config:
  32. biggest_first = False
  33. iec = False
  34. size_limit = 0.01
  35. colors = ['#e60004', '#f28e00', '#ffed00', '#940084',
  36. '#2e1d86', '#0068b5', '#009836', '#97c000']
  37. #
  38. # This function adds a new file to 'filesdict', after checking its
  39. # size. The 'filesdict' contain the relative path of the file as the
  40. # key, and as the value a tuple containing the name of the package to
  41. # which the file belongs and the size of the file.
  42. #
  43. # filesdict: the dict to which the file is added
  44. # relpath: relative path of the file
  45. # fullpath: absolute path to the file
  46. # pkg: package to which the file belongs
  47. #
  48. def add_file(filesdict, relpath, abspath, pkg):
  49. if not os.path.exists(abspath):
  50. return
  51. if os.path.islink(abspath):
  52. return
  53. sz = os.stat(abspath).st_size
  54. filesdict[relpath] = (pkg, sz)
  55. #
  56. # This function returns a dict where each key is the path of a file in
  57. # the root filesystem, and the value is a tuple containing two
  58. # elements: the name of the package to which this file belongs and the
  59. # size of the file.
  60. #
  61. # builddir: path to the Buildroot output directory
  62. #
  63. def build_package_dict(builddir):
  64. filesdict = {}
  65. with open(os.path.join(builddir, "build", "packages-file-list.txt")) as f:
  66. for l in f.readlines():
  67. pkg, fpath = l.split(",", 1)
  68. # remove the initial './' in each file path
  69. fpath = fpath.strip()[2:]
  70. fullpath = os.path.join(builddir, "target", fpath)
  71. add_file(filesdict, fpath, fullpath, pkg)
  72. return filesdict
  73. #
  74. # This function builds a dictionary that contains the name of a
  75. # package as key, and the size of the files installed by this package
  76. # as the value.
  77. #
  78. # filesdict: dictionary with the name of the files as key, and as
  79. # value a tuple containing the name of the package to which the files
  80. # belongs, and the size of the file. As returned by
  81. # build_package_dict.
  82. #
  83. # builddir: path to the Buildroot output directory
  84. #
  85. def build_package_size(filesdict, builddir):
  86. pkgsize = collections.defaultdict(int)
  87. seeninodes = set()
  88. for root, _, files in os.walk(os.path.join(builddir, "target")):
  89. for f in files:
  90. fpath = os.path.join(root, f)
  91. if os.path.islink(fpath):
  92. continue
  93. st = os.stat(fpath)
  94. if st.st_ino in seeninodes:
  95. # hard link
  96. continue
  97. else:
  98. seeninodes.add(st.st_ino)
  99. frelpath = os.path.relpath(fpath, os.path.join(builddir, "target"))
  100. if frelpath not in filesdict:
  101. print("WARNING: %s is not part of any package" % frelpath)
  102. pkg = "unknown"
  103. else:
  104. pkg = filesdict[frelpath][0]
  105. pkgsize[pkg] += st.st_size
  106. return pkgsize
  107. #
  108. # Given a dict returned by build_package_size(), this function
  109. # generates a pie chart of the size installed by each package.
  110. #
  111. # pkgsize: dictionary with the name of the package as a key, and the
  112. # size as the value, as returned by build_package_size.
  113. #
  114. # outputf: output file for the graph
  115. #
  116. def draw_graph(pkgsize, outputf):
  117. def size2string(sz):
  118. if Config.iec:
  119. divider = 1024.0
  120. prefixes = ['', 'Ki', 'Mi', 'Gi', 'Ti']
  121. else:
  122. divider = 1000.0
  123. prefixes = ['', 'k', 'M', 'G', 'T']
  124. while sz > divider and len(prefixes) > 1:
  125. prefixes = prefixes[1:]
  126. sz = sz/divider
  127. # precision is made so that there are always at least three meaningful
  128. # digits displayed (e.g. '3.14' and '10.4', not just '3' and '10')
  129. precision = int(2-math.floor(math.log10(sz))) if sz < 1000 else 0
  130. return '{:.{prec}f} {}B'.format(sz, prefixes[0], prec=precision)
  131. total = sum(pkgsize.values())
  132. labels = []
  133. values = []
  134. other_value = 0
  135. unknown_value = 0
  136. for (p, sz) in sorted(pkgsize.items(), key=lambda x: x[1],
  137. reverse=Config.biggest_first):
  138. if sz < (total * Config.size_limit):
  139. other_value += sz
  140. elif p == "unknown":
  141. unknown_value = sz
  142. else:
  143. labels.append("%s (%s)" % (p, size2string(sz)))
  144. values.append(sz)
  145. if unknown_value != 0:
  146. labels.append("Unknown (%s)" % (size2string(unknown_value)))
  147. values.append(unknown_value)
  148. if other_value != 0:
  149. labels.append("Other (%s)" % (size2string(other_value)))
  150. values.append(other_value)
  151. plt.figure()
  152. patches, texts, autotexts = plt.pie(values, labels=labels,
  153. autopct='%1.1f%%', shadow=True,
  154. colors=Config.colors)
  155. # Reduce text size
  156. proptease = fm.FontProperties()
  157. proptease.set_size('xx-small')
  158. plt.setp(autotexts, fontproperties=proptease)
  159. plt.setp(texts, fontproperties=proptease)
  160. plt.suptitle("Filesystem size per package", fontsize=18, y=.97)
  161. plt.title("Total filesystem size: %s" % (size2string(total)), fontsize=10,
  162. y=.96)
  163. plt.savefig(outputf)
  164. #
  165. # Generate a CSV file with statistics about the size of each file, its
  166. # size contribution to the package and to the overall system.
  167. #
  168. # filesdict: dictionary with the name of the files as key, and as
  169. # value a tuple containing the name of the package to which the files
  170. # belongs, and the size of the file. As returned by
  171. # build_package_dict.
  172. #
  173. # pkgsize: dictionary with the name of the package as a key, and the
  174. # size as the value, as returned by build_package_size.
  175. #
  176. # outputf: output CSV file
  177. #
  178. def gen_files_csv(filesdict, pkgsizes, outputf):
  179. total = 0
  180. for (p, sz) in pkgsizes.items():
  181. total += sz
  182. with open(outputf, 'w') as csvfile:
  183. wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
  184. wr.writerow(["File name",
  185. "Package name",
  186. "File size",
  187. "Package size",
  188. "File size in package (%)",
  189. "File size in system (%)"])
  190. for f, (pkgname, filesize) in filesdict.items():
  191. pkgsize = pkgsizes[pkgname]
  192. if pkgsize == 0:
  193. percent_pkg = 0
  194. else:
  195. percent_pkg = float(filesize) / pkgsize * 100
  196. percent_total = float(filesize) / total * 100
  197. wr.writerow([f, pkgname, filesize, pkgsize,
  198. "%.1f" % percent_pkg,
  199. "%.1f" % percent_total])
  200. #
  201. # Generate a CSV file with statistics about the size of each package,
  202. # and their size contribution to the overall system.
  203. #
  204. # pkgsize: dictionary with the name of the package as a key, and the
  205. # size as the value, as returned by build_package_size.
  206. #
  207. # outputf: output CSV file
  208. #
  209. def gen_packages_csv(pkgsizes, outputf):
  210. total = sum(pkgsizes.values())
  211. with open(outputf, 'w') as csvfile:
  212. wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
  213. wr.writerow(["Package name", "Package size",
  214. "Package size in system (%)"])
  215. for (pkg, size) in pkgsizes.items():
  216. wr.writerow([pkg, size, "%.1f" % (float(size) / total * 100)])
  217. #
  218. # Our special action for --iec, --binary, --si, --decimal
  219. #
  220. class PrefixAction(argparse.Action):
  221. def __init__(self, option_strings, dest, **kwargs):
  222. for key in ["type", "nargs"]:
  223. if key in kwargs:
  224. raise ValueError('"{}" not allowed'.format(key))
  225. super(PrefixAction, self).__init__(option_strings, dest, nargs=0,
  226. type=bool, **kwargs)
  227. def __call__(self, parser, namespace, values, option_string=None):
  228. setattr(namespace, self.dest, option_string in ["--iec", "--binary"])
  229. def main():
  230. parser = argparse.ArgumentParser(description='Draw size statistics graphs')
  231. parser.add_argument("--builddir", '-i', metavar="BUILDDIR", required=True,
  232. help="Buildroot output directory")
  233. parser.add_argument("--graph", '-g', metavar="GRAPH",
  234. help="Graph output file (.pdf or .png extension)")
  235. parser.add_argument("--file-size-csv", '-f', metavar="FILE_SIZE_CSV",
  236. help="CSV output file with file size statistics")
  237. parser.add_argument("--package-size-csv", '-p', metavar="PKG_SIZE_CSV",
  238. help="CSV output file with package size statistics")
  239. parser.add_argument("--biggest-first", action='store_true',
  240. help="Sort packages in decreasing size order, " +
  241. "rather than in increasing size order")
  242. parser.add_argument("--iec", "--binary", "--si", "--decimal",
  243. action=PrefixAction,
  244. help="Use IEC (binary, powers of 1024) or SI (decimal, "
  245. "powers of 1000, the default) prefixes")
  246. parser.add_argument("--size-limit", "-l", type=float,
  247. help='Under this size ratio, files are accounted to ' +
  248. 'the generic "Other" package. Default: 0.01 (1%%)')
  249. args = parser.parse_args()
  250. Config.biggest_first = args.biggest_first
  251. Config.iec = args.iec
  252. if args.size_limit is not None:
  253. if args.size_limit < 0.0 or args.size_limit > 1.0:
  254. parser.error("--size-limit must be in [0.0..1.0]")
  255. Config.size_limit = args.size_limit
  256. # Find out which package installed what files
  257. pkgdict = build_package_dict(args.builddir)
  258. # Collect the size installed by each package
  259. pkgsize = build_package_size(pkgdict, args.builddir)
  260. if args.graph:
  261. draw_graph(pkgsize, args.graph)
  262. if args.file_size_csv:
  263. gen_files_csv(pkgdict, pkgsize, args.file_size_csv)
  264. if args.package_size_csv:
  265. gen_packages_csv(pkgsize, args.package_size_csv)
  266. if __name__ == "__main__":
  267. main()