size-stats 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240
  1. #!/usr/bin/env python
  2. # Copyright (C) 2014 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
  3. # This program is free software; you can redistribute it and/or modify
  4. # it under the terms of the GNU General Public License as published by
  5. # the Free Software Foundation; either version 2 of the License, or
  6. # (at your option) any later version.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. # General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License
  14. # along with this program; if not, write to the Free Software
  15. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  16. import sys
  17. import os
  18. import os.path
  19. import argparse
  20. import csv
  21. import collections
  22. try:
  23. import matplotlib
  24. matplotlib.use('Agg')
  25. import matplotlib.font_manager as fm
  26. import matplotlib.pyplot as plt
  27. except ImportError:
  28. sys.stderr.write("You need python-matplotlib to generate the size graph\n")
  29. exit(1)
  30. colors = ['#e60004', '#009836', '#2e1d86', '#ffed00',
  31. '#0068b5', '#f28e00', '#940084', '#97c000']
  32. #
  33. # This function adds a new file to 'filesdict', after checking its
  34. # size. The 'filesdict' contain the relative path of the file as the
  35. # key, and as the value a tuple containing the name of the package to
  36. # which the file belongs and the size of the file.
  37. #
  38. # filesdict: the dict to which the file is added
  39. # relpath: relative path of the file
  40. # fullpath: absolute path to the file
  41. # pkg: package to which the file belongs
  42. #
  43. def add_file(filesdict, relpath, abspath, pkg):
  44. if not os.path.exists(abspath):
  45. return
  46. if os.path.islink(abspath):
  47. return
  48. sz = os.stat(abspath).st_size
  49. filesdict[relpath] = (pkg, sz)
  50. #
  51. # This function returns a dict where each key is the path of a file in
  52. # the root filesystem, and the value is a tuple containing two
  53. # elements: the name of the package to which this file belongs and the
  54. # size of the file.
  55. #
  56. # builddir: path to the Buildroot output directory
  57. #
  58. def build_package_dict(builddir):
  59. filesdict = {}
  60. with open(os.path.join(builddir, "build", "packages-file-list.txt")) as filelistf:
  61. for l in filelistf.readlines():
  62. pkg, fpath = l.split(",", 1)
  63. # remove the initial './' in each file path
  64. fpath = fpath.strip()[2:]
  65. fullpath = os.path.join(builddir, "target", fpath)
  66. add_file(filesdict, fpath, fullpath, pkg)
  67. return filesdict
  68. #
  69. # This function builds a dictionary that contains the name of a
  70. # package as key, and the size of the files installed by this package
  71. # as the value.
  72. #
  73. # filesdict: dictionary with the name of the files as key, and as
  74. # value a tuple containing the name of the package to which the files
  75. # belongs, and the size of the file. As returned by
  76. # build_package_dict.
  77. #
  78. # builddir: path to the Buildroot output directory
  79. #
  80. def build_package_size(filesdict, builddir):
  81. pkgsize = collections.defaultdict(int)
  82. seeninodes = set()
  83. for root, _, files in os.walk(os.path.join(builddir, "target")):
  84. for f in files:
  85. fpath = os.path.join(root, f)
  86. if os.path.islink(fpath):
  87. continue
  88. st = os.stat(fpath)
  89. if st.st_ino in seeninodes:
  90. # hard link
  91. continue
  92. else:
  93. seeninodes.add(st.st_ino)
  94. frelpath = os.path.relpath(fpath, os.path.join(builddir, "target"))
  95. if frelpath not in filesdict:
  96. print("WARNING: %s is not part of any package" % frelpath)
  97. pkg = "unknown"
  98. else:
  99. pkg = filesdict[frelpath][0]
  100. pkgsize[pkg] += st.st_size
  101. return pkgsize
  102. #
  103. # Given a dict returned by build_package_size(), this function
  104. # generates a pie chart of the size installed by each package.
  105. #
  106. # pkgsize: dictionary with the name of the package as a key, and the
  107. # size as the value, as returned by build_package_size.
  108. #
  109. # outputf: output file for the graph
  110. #
  111. def draw_graph(pkgsize, outputf):
  112. total = sum(pkgsize.values())
  113. labels = []
  114. values = []
  115. other_value = 0
  116. for (p, sz) in sorted(pkgsize.items(), key=lambda x: x[1]):
  117. if sz < (total * 0.01):
  118. other_value += sz
  119. else:
  120. labels.append("%s (%d kB)" % (p, sz / 1000.))
  121. values.append(sz)
  122. labels.append("Other (%d kB)" % (other_value / 1000.))
  123. values.append(other_value)
  124. plt.figure()
  125. patches, texts, autotexts = plt.pie(values, labels=labels,
  126. autopct='%1.1f%%', shadow=True,
  127. colors=colors)
  128. # Reduce text size
  129. proptease = fm.FontProperties()
  130. proptease.set_size('xx-small')
  131. plt.setp(autotexts, fontproperties=proptease)
  132. plt.setp(texts, fontproperties=proptease)
  133. plt.suptitle("Filesystem size per package", fontsize=18, y=.97)
  134. plt.title("Total filesystem size: %d kB" % (total / 1000.), fontsize=10, y=.96)
  135. plt.savefig(outputf)
  136. #
  137. # Generate a CSV file with statistics about the size of each file, its
  138. # size contribution to the package and to the overall system.
  139. #
  140. # filesdict: dictionary with the name of the files as key, and as
  141. # value a tuple containing the name of the package to which the files
  142. # belongs, and the size of the file. As returned by
  143. # build_package_dict.
  144. #
  145. # pkgsize: dictionary with the name of the package as a key, and the
  146. # size as the value, as returned by build_package_size.
  147. #
  148. # outputf: output CSV file
  149. #
  150. def gen_files_csv(filesdict, pkgsizes, outputf):
  151. total = 0
  152. for (p, sz) in pkgsizes.items():
  153. total += sz
  154. with open(outputf, 'w') as csvfile:
  155. wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
  156. wr.writerow(["File name",
  157. "Package name",
  158. "File size",
  159. "Package size",
  160. "File size in package (%)",
  161. "File size in system (%)"])
  162. for f, (pkgname, filesize) in filesdict.items():
  163. pkgsize = pkgsizes[pkgname]
  164. if pkgsize == 0:
  165. percent_pkg = 0
  166. else:
  167. percent_pkg = float(filesize) / pkgsize * 100
  168. percent_total = float(filesize) / total * 100
  169. wr.writerow([f, pkgname, filesize, pkgsize,
  170. "%.1f" % percent_pkg,
  171. "%.1f" % percent_total])
  172. #
  173. # Generate a CSV file with statistics about the size of each package,
  174. # and their size contribution to the overall system.
  175. #
  176. # pkgsize: dictionary with the name of the package as a key, and the
  177. # size as the value, as returned by build_package_size.
  178. #
  179. # outputf: output CSV file
  180. #
  181. def gen_packages_csv(pkgsizes, outputf):
  182. total = sum(pkgsizes.values())
  183. with open(outputf, 'w') as csvfile:
  184. wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
  185. wr.writerow(["Package name", "Package size", "Package size in system (%)"])
  186. for (pkg, size) in pkgsizes.items():
  187. wr.writerow([pkg, size, "%.1f" % (float(size) / total * 100)])
  188. parser = argparse.ArgumentParser(description='Draw size statistics graphs')
  189. parser.add_argument("--builddir", '-i', metavar="BUILDDIR", required=True,
  190. help="Buildroot output directory")
  191. parser.add_argument("--graph", '-g', metavar="GRAPH",
  192. help="Graph output file (.pdf or .png extension)")
  193. parser.add_argument("--file-size-csv", '-f', metavar="FILE_SIZE_CSV",
  194. help="CSV output file with file size statistics")
  195. parser.add_argument("--package-size-csv", '-p', metavar="PKG_SIZE_CSV",
  196. help="CSV output file with package size statistics")
  197. args = parser.parse_args()
  198. # Find out which package installed what files
  199. pkgdict = build_package_dict(args.builddir)
  200. # Collect the size installed by each package
  201. pkgsize = build_package_size(pkgdict, args.builddir)
  202. if args.graph:
  203. draw_graph(pkgsize, args.graph)
  204. if args.file_size_csv:
  205. gen_files_csv(pkgdict, pkgsize, args.file_size_csv)
  206. if args.package_size_csv:
  207. gen_packages_csv(pkgsize, args.package_size_csv)