moveconfig.py 62 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813
  1. #!/usr/bin/env python3
  2. # SPDX-License-Identifier: GPL-2.0+
  3. #
  4. # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
  5. #
  6. """
  7. Move config options from headers to defconfig files.
  8. See doc/develop/moveconfig.rst for documentation.
  9. """
  10. from argparse import ArgumentParser
  11. import asteval
  12. import collections
  13. from contextlib import ExitStack
  14. import copy
  15. import difflib
  16. import doctest
  17. import filecmp
  18. import fnmatch
  19. import glob
  20. import multiprocessing
  21. import os
  22. import queue
  23. import re
  24. import shutil
  25. import subprocess
  26. import sys
  27. import tempfile
  28. import threading
  29. import time
  30. import unittest
  31. from buildman import bsettings
  32. from buildman import kconfiglib
  33. from buildman import toolchain
  34. SHOW_GNU_MAKE = 'scripts/show-gnu-make'
  35. SLEEP_TIME=0.03
  36. STATE_IDLE = 0
  37. STATE_DEFCONFIG = 1
  38. STATE_AUTOCONF = 2
  39. STATE_SAVEDEFCONFIG = 3
  40. ACTION_MOVE = 0
  41. ACTION_NO_ENTRY = 1
  42. ACTION_NO_ENTRY_WARN = 2
  43. ACTION_NO_CHANGE = 3
  44. COLOR_BLACK = '0;30'
  45. COLOR_RED = '0;31'
  46. COLOR_GREEN = '0;32'
  47. COLOR_BROWN = '0;33'
  48. COLOR_BLUE = '0;34'
  49. COLOR_PURPLE = '0;35'
  50. COLOR_CYAN = '0;36'
  51. COLOR_LIGHT_GRAY = '0;37'
  52. COLOR_DARK_GRAY = '1;30'
  53. COLOR_LIGHT_RED = '1;31'
  54. COLOR_LIGHT_GREEN = '1;32'
  55. COLOR_YELLOW = '1;33'
  56. COLOR_LIGHT_BLUE = '1;34'
  57. COLOR_LIGHT_PURPLE = '1;35'
  58. COLOR_LIGHT_CYAN = '1;36'
  59. COLOR_WHITE = '1;37'
  60. AUTO_CONF_PATH = 'include/config/auto.conf'
  61. CONFIG_DATABASE = 'moveconfig.db'
  62. CONFIG_LEN = len('CONFIG_')
  63. SIZES = {
  64. 'SZ_1': 0x00000001, 'SZ_2': 0x00000002,
  65. 'SZ_4': 0x00000004, 'SZ_8': 0x00000008,
  66. 'SZ_16': 0x00000010, 'SZ_32': 0x00000020,
  67. 'SZ_64': 0x00000040, 'SZ_128': 0x00000080,
  68. 'SZ_256': 0x00000100, 'SZ_512': 0x00000200,
  69. 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800,
  70. 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000,
  71. 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000,
  72. 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000,
  73. 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000,
  74. 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000,
  75. 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000,
  76. 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000,
  77. 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000,
  78. 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000,
  79. 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000,
  80. 'SZ_4G': 0x100000000
  81. }
  82. ### helper functions ###
  83. def check_top_directory():
  84. """Exit if we are not at the top of source directory."""
  85. for fname in 'README', 'Licenses':
  86. if not os.path.exists(fname):
  87. sys.exit('Please run at the top of source directory.')
  88. def check_clean_directory():
  89. """Exit if the source tree is not clean."""
  90. for fname in '.config', 'include/config':
  91. if os.path.exists(fname):
  92. sys.exit("source tree is not clean, please run 'make mrproper'")
  93. def get_make_cmd():
  94. """Get the command name of GNU Make.
  95. U-Boot needs GNU Make for building, but the command name is not
  96. necessarily "make". (for example, "gmake" on FreeBSD).
  97. Returns the most appropriate command name on your system.
  98. """
  99. with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc:
  100. ret = proc.communicate()
  101. if proc.returncode:
  102. sys.exit('GNU Make not found')
  103. return ret[0].rstrip()
  104. def get_matched_defconfig(line):
  105. """Get the defconfig files that match a pattern
  106. Args:
  107. line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or
  108. 'k2*_defconfig'. If no directory is provided, 'configs/' is
  109. prepended
  110. Returns:
  111. list of str: a list of matching defconfig files
  112. """
  113. dirname = os.path.dirname(line)
  114. if dirname:
  115. pattern = line
  116. else:
  117. pattern = os.path.join('configs', line)
  118. return glob.glob(pattern) + glob.glob(pattern + '_defconfig')
  119. def get_matched_defconfigs(defconfigs_file):
  120. """Get all the defconfig files that match the patterns in a file.
  121. Args:
  122. defconfigs_file (str): File containing a list of defconfigs to process,
  123. or '-' to read the list from stdin
  124. Returns:
  125. list of str: A list of paths to defconfig files, with no duplicates
  126. """
  127. defconfigs = []
  128. with ExitStack() as stack:
  129. if defconfigs_file == '-':
  130. inf = sys.stdin
  131. defconfigs_file = 'stdin'
  132. else:
  133. inf = stack.enter_context(open(defconfigs_file, encoding='utf-8'))
  134. for i, line in enumerate(inf):
  135. line = line.strip()
  136. if not line:
  137. continue # skip blank lines silently
  138. if ' ' in line:
  139. line = line.split(' ')[0] # handle 'git log' input
  140. matched = get_matched_defconfig(line)
  141. if not matched:
  142. print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'",
  143. file=sys.stderr)
  144. defconfigs += matched
  145. # use set() to drop multiple matching
  146. return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)]
  147. def get_all_defconfigs():
  148. """Get all the defconfig files under the configs/ directory.
  149. Returns:
  150. list of str: List of paths to defconfig files
  151. """
  152. defconfigs = []
  153. for (dirpath, _, filenames) in os.walk('configs'):
  154. dirpath = dirpath[len('configs') + 1:]
  155. for filename in fnmatch.filter(filenames, '*_defconfig'):
  156. defconfigs.append(os.path.join(dirpath, filename))
  157. return defconfigs
  158. def color_text(color_enabled, color, string):
  159. """Return colored string."""
  160. if color_enabled:
  161. # LF should not be surrounded by the escape sequence.
  162. # Otherwise, additional whitespace or line-feed might be printed.
  163. return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
  164. for s in string.split('\n') ])
  165. return string
  166. def show_diff(alines, blines, file_path, color_enabled):
  167. """Show unidified diff.
  168. Args:
  169. alines (list of str): A list of lines (before)
  170. blines (list of str): A list of lines (after)
  171. file_path (str): Path to the file
  172. color_enabled (bool): Display the diff in color
  173. """
  174. diff = difflib.unified_diff(alines, blines,
  175. fromfile=os.path.join('a', file_path),
  176. tofile=os.path.join('b', file_path))
  177. for line in diff:
  178. if line.startswith('-') and not line.startswith('--'):
  179. print(color_text(color_enabled, COLOR_RED, line))
  180. elif line.startswith('+') and not line.startswith('++'):
  181. print(color_text(color_enabled, COLOR_GREEN, line))
  182. else:
  183. print(line)
  184. def extend_matched_lines(lines, matched, pre_patterns, post_patterns,
  185. extend_pre, extend_post):
  186. """Extend matched lines if desired patterns are found before/after already
  187. matched lines.
  188. Args:
  189. lines (list of str): list of lines handled.
  190. matched (list of int): list of line numbers that have been already
  191. matched (will be updated by this function)
  192. pre_patterns (list of re.Pattern): list of regular expression that should
  193. be matched as preamble
  194. post_patterns (list of re.Pattern): list of regular expression that should
  195. be matched as postamble
  196. extend_pre (bool): Add the line number of matched preamble to the matched
  197. list
  198. extend_post (bool): Add the line number of matched postamble to the
  199. matched list
  200. """
  201. extended_matched = []
  202. j = matched[0]
  203. for i in matched:
  204. if i == 0 or i < j:
  205. continue
  206. j = i
  207. while j in matched:
  208. j += 1
  209. if j >= len(lines):
  210. break
  211. for pat in pre_patterns:
  212. if pat.search(lines[i - 1]):
  213. break
  214. else:
  215. # not matched
  216. continue
  217. for pat in post_patterns:
  218. if pat.search(lines[j]):
  219. break
  220. else:
  221. # not matched
  222. continue
  223. if extend_pre:
  224. extended_matched.append(i - 1)
  225. if extend_post:
  226. extended_matched.append(j)
  227. matched += extended_matched
  228. matched.sort()
  229. def confirm(args, prompt):
  230. """Ask the user to confirm something
  231. Args:
  232. args (Namespace ): program arguments
  233. Returns:
  234. bool: True to confirm, False to cancel/stop
  235. """
  236. if not args.yes:
  237. while True:
  238. choice = input(f'{prompt} [y/n]: ')
  239. choice = choice.lower()
  240. print(choice)
  241. if choice in ('y', 'n'):
  242. break
  243. if choice == 'n':
  244. return False
  245. return True
  246. def write_file(fname, data):
  247. """Write data to a file
  248. Args:
  249. fname (str): Filename to write to
  250. data (list of str): Lines to write (with or without trailing newline);
  251. or str to write
  252. """
  253. with open(fname, 'w', encoding='utf-8') as out:
  254. if isinstance(data, list):
  255. for line in data:
  256. print(line.rstrip('\n'), file=out)
  257. else:
  258. out.write(data)
  259. def read_file(fname, as_lines=True, skip_unicode=False):
  260. """Read a file and return the contents
  261. Args:
  262. fname (str): Filename to read from
  263. as_lines: Return file contents as a list of lines
  264. skip_unicode (bool): True to report unicode errors and continue
  265. Returns:
  266. iter of str: List of ;ines from the file with newline removed; str if
  267. as_lines is False with newlines intact; or None if a unicode error
  268. occurred
  269. Raises:
  270. UnicodeDecodeError: Unicode error occurred when reading
  271. """
  272. with open(fname, encoding='utf-8') as inf:
  273. try:
  274. if as_lines:
  275. return [line.rstrip('\n') for line in inf.readlines()]
  276. else:
  277. return inf.read()
  278. except UnicodeDecodeError as e:
  279. if not skip_unicode:
  280. raises
  281. print("Failed on file %s': %s" % (fname, e))
  282. return None
  283. def cleanup_empty_blocks(header_path, args):
  284. """Clean up empty conditional blocks
  285. Args:
  286. header_path (str): path to the cleaned file.
  287. args (Namespace): program arguments
  288. """
  289. pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M)
  290. data = read_file(header_path, as_lines=False, skip_unicode=True)
  291. if data is None:
  292. return
  293. new_data = pattern.sub('\n', data)
  294. show_diff(data.splitlines(True), new_data.splitlines(True), header_path,
  295. args.color)
  296. if args.dry_run:
  297. return
  298. if new_data != data:
  299. write_file(header_path, new_data)
  300. def cleanup_one_header(header_path, patterns, args):
  301. """Clean regex-matched lines away from a file.
  302. Args:
  303. header_path: path to the cleaned file.
  304. patterns: list of regex patterns. Any lines matching to these
  305. patterns are deleted.
  306. args (Namespace): program arguments
  307. """
  308. lines = read_file(header_path, skip_unicode=True)
  309. if lines is None:
  310. return
  311. matched = []
  312. for i, line in enumerate(lines):
  313. if i - 1 in matched and lines[i - 1].endswith('\\'):
  314. matched.append(i)
  315. continue
  316. for pattern in patterns:
  317. if pattern.search(line):
  318. matched.append(i)
  319. break
  320. if not matched:
  321. return
  322. # remove empty #ifdef ... #endif, successive blank lines
  323. pattern_if = re.compile(r'#\s*if(def|ndef)?\b') # #if, #ifdef, #ifndef
  324. pattern_elif = re.compile(r'#\s*el(if|se)\b') # #elif, #else
  325. pattern_endif = re.compile(r'#\s*endif\b') # #endif
  326. pattern_blank = re.compile(r'^\s*$') # empty line
  327. while True:
  328. old_matched = copy.copy(matched)
  329. extend_matched_lines(lines, matched, [pattern_if],
  330. [pattern_endif], True, True)
  331. extend_matched_lines(lines, matched, [pattern_elif],
  332. [pattern_elif, pattern_endif], True, False)
  333. extend_matched_lines(lines, matched, [pattern_if, pattern_elif],
  334. [pattern_blank], False, True)
  335. extend_matched_lines(lines, matched, [pattern_blank],
  336. [pattern_elif, pattern_endif], True, False)
  337. extend_matched_lines(lines, matched, [pattern_blank],
  338. [pattern_blank], True, False)
  339. if matched == old_matched:
  340. break
  341. tolines = copy.copy(lines)
  342. for i in reversed(matched):
  343. tolines.pop(i)
  344. show_diff(lines, tolines, header_path, args.color)
  345. if args.dry_run:
  346. return
  347. write_file(header_path, tolines)
  348. def cleanup_headers(configs, args):
  349. """Delete config defines from board headers.
  350. Args:
  351. configs: A list of CONFIGs to remove.
  352. args (Namespace): program arguments
  353. """
  354. if not confirm(args, 'Clean up headers?'):
  355. return
  356. patterns = []
  357. for config in configs:
  358. patterns.append(re.compile(r'#\s*define\s+%s\b' % config))
  359. patterns.append(re.compile(r'#\s*undef\s+%s\b' % config))
  360. for dir in 'include', 'arch', 'board':
  361. for (dirpath, dirnames, filenames) in os.walk(dir):
  362. if dirpath == os.path.join('include', 'generated'):
  363. continue
  364. for filename in filenames:
  365. if not filename.endswith(('~', '.dts', '.dtsi', '.bin',
  366. '.elf','.aml','.dat')):
  367. header_path = os.path.join(dirpath, filename)
  368. # This file contains UTF-16 data and no CONFIG symbols
  369. if header_path == 'include/video_font_data.h':
  370. continue
  371. cleanup_one_header(header_path, patterns, args)
  372. cleanup_empty_blocks(header_path, args)
  373. def cleanup_one_extra_option(defconfig_path, configs, args):
  374. """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file.
  375. Args:
  376. defconfig_path: path to the cleaned defconfig file.
  377. configs: A list of CONFIGs to remove.
  378. args (Namespace): program arguments
  379. """
  380. start = 'CONFIG_SYS_EXTRA_OPTIONS="'
  381. end = '"'
  382. lines = read_file(defconfig_path)
  383. for i, line in enumerate(lines):
  384. if line.startswith(start) and line.endswith(end):
  385. break
  386. else:
  387. # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig
  388. return
  389. old_tokens = line[len(start):-len(end)].split(',')
  390. new_tokens = []
  391. for token in old_tokens:
  392. pos = token.find('=')
  393. if not (token[:pos] if pos >= 0 else token) in configs:
  394. new_tokens.append(token)
  395. if new_tokens == old_tokens:
  396. return
  397. tolines = copy.copy(lines)
  398. if new_tokens:
  399. tolines[i] = start + ','.join(new_tokens) + end
  400. else:
  401. tolines.pop(i)
  402. show_diff(lines, tolines, defconfig_path, args.color)
  403. if args.dry_run:
  404. return
  405. write_file(defconfig_path, tolines)
  406. def cleanup_extra_options(configs, args):
  407. """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files.
  408. Args:
  409. configs: A list of CONFIGs to remove.
  410. args (Namespace): program arguments
  411. """
  412. if not confirm(args, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'):
  413. return
  414. configs = [ config[len('CONFIG_'):] for config in configs ]
  415. defconfigs = get_all_defconfigs()
  416. for defconfig in defconfigs:
  417. cleanup_one_extra_option(os.path.join('configs', defconfig), configs,
  418. args)
  419. def cleanup_whitelist(configs, args):
  420. """Delete config whitelist entries
  421. Args:
  422. configs: A list of CONFIGs to remove.
  423. args (Namespace): program arguments
  424. """
  425. if not confirm(args, 'Clean up whitelist entries?'):
  426. return
  427. lines = read_file(os.path.join('scripts', 'config_whitelist.txt'))
  428. lines = [x for x in lines if x.strip() not in configs]
  429. write_file(os.path.join('scripts', 'config_whitelist.txt'), lines)
  430. def find_matching(patterns, line):
  431. for pat in patterns:
  432. if pat.search(line):
  433. return True
  434. return False
  435. def cleanup_readme(configs, args):
  436. """Delete config description in README
  437. Args:
  438. configs: A list of CONFIGs to remove.
  439. args (Namespace): program arguments
  440. """
  441. if not confirm(args, 'Clean up README?'):
  442. return
  443. patterns = []
  444. for config in configs:
  445. patterns.append(re.compile(r'^\s+%s' % config))
  446. lines = read_file('README')
  447. found = False
  448. newlines = []
  449. for line in lines:
  450. if not found:
  451. found = find_matching(patterns, line)
  452. if found:
  453. continue
  454. if found and re.search(r'^\s+CONFIG', line):
  455. found = False
  456. if not found:
  457. newlines.append(line)
  458. write_file('README', newlines)
  459. def try_expand(line):
  460. """If value looks like an expression, try expanding it
  461. Otherwise just return the existing value
  462. """
  463. if line.find('=') == -1:
  464. return line
  465. try:
  466. aeval = asteval.Interpreter( usersyms=SIZES, minimal=True )
  467. cfg, val = re.split("=", line)
  468. val= val.strip('\"')
  469. if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val):
  470. newval = hex(aeval(val))
  471. print('\tExpanded expression %s to %s' % (val, newval))
  472. return cfg+'='+newval
  473. except:
  474. print('\tFailed to expand expression in %s' % line)
  475. return line
  476. ### classes ###
  477. class Progress:
  478. """Progress Indicator"""
  479. def __init__(self, total):
  480. """Create a new progress indicator.
  481. Args:
  482. total: A number of defconfig files to process.
  483. """
  484. self.current = 0
  485. self.total = total
  486. def inc(self):
  487. """Increment the number of processed defconfig files."""
  488. self.current += 1
  489. def show(self):
  490. """Display the progress."""
  491. print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ')
  492. sys.stdout.flush()
  493. class KconfigScanner:
  494. """Kconfig scanner."""
  495. def __init__(self):
  496. """Scan all the Kconfig files and create a Config object."""
  497. # Define environment variables referenced from Kconfig
  498. os.environ['srctree'] = os.getcwd()
  499. os.environ['UBOOTVERSION'] = 'dummy'
  500. os.environ['KCONFIG_OBJDIR'] = ''
  501. self.conf = kconfiglib.Kconfig()
  502. class KconfigParser:
  503. """A parser of .config and include/autoconf.mk."""
  504. re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
  505. re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
  506. def __init__(self, configs, args, build_dir):
  507. """Create a new parser.
  508. Args:
  509. configs: A list of CONFIGs to move.
  510. args (Namespace): program arguments
  511. build_dir: Build directory.
  512. """
  513. self.configs = configs
  514. self.args = args
  515. self.dotconfig = os.path.join(build_dir, '.config')
  516. self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
  517. self.spl_autoconf = os.path.join(build_dir, 'spl', 'include',
  518. 'autoconf.mk')
  519. self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH)
  520. self.defconfig = os.path.join(build_dir, 'defconfig')
  521. def get_arch(self):
  522. """Parse .config file and return the architecture.
  523. Returns:
  524. Architecture name (e.g. 'arm').
  525. """
  526. arch = ''
  527. cpu = ''
  528. for line in read_file(self.dotconfig):
  529. m = self.re_arch.match(line)
  530. if m:
  531. arch = m.group(1)
  532. continue
  533. m = self.re_cpu.match(line)
  534. if m:
  535. cpu = m.group(1)
  536. if not arch:
  537. return None
  538. # fix-up for aarch64
  539. if arch == 'arm' and cpu == 'armv8':
  540. arch = 'aarch64'
  541. return arch
  542. def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
  543. """Parse .config, defconfig, include/autoconf.mk for one config.
  544. This function looks for the config options in the lines from
  545. defconfig, .config, and include/autoconf.mk in order to decide
  546. which action should be taken for this defconfig.
  547. Args:
  548. config: CONFIG name to parse.
  549. dotconfig_lines: lines from the .config file.
  550. autoconf_lines: lines from the include/autoconf.mk file.
  551. Returns:
  552. A tupple of the action for this defconfig and the line
  553. matched for the config.
  554. """
  555. not_set = '# %s is not set' % config
  556. for line in autoconf_lines:
  557. line = line.rstrip()
  558. if line.startswith(config + '='):
  559. new_val = line
  560. break
  561. else:
  562. new_val = not_set
  563. new_val = try_expand(new_val)
  564. for line in dotconfig_lines:
  565. line = line.rstrip()
  566. if line.startswith(config + '=') or line == not_set:
  567. old_val = line
  568. break
  569. else:
  570. if new_val == not_set:
  571. return (ACTION_NO_ENTRY, config)
  572. else:
  573. return (ACTION_NO_ENTRY_WARN, config)
  574. # If this CONFIG is neither bool nor trisate
  575. if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
  576. # tools/scripts/define2mk.sed changes '1' to 'y'.
  577. # This is a problem if the CONFIG is int type.
  578. # Check the type in Kconfig and handle it correctly.
  579. if new_val[-2:] == '=y':
  580. new_val = new_val[:-1] + '1'
  581. return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE,
  582. new_val)
  583. def update_dotconfig(self):
  584. """Parse files for the config options and update the .config.
  585. This function parses the generated .config and include/autoconf.mk
  586. searching the target options.
  587. Move the config option(s) to the .config as needed.
  588. Args:
  589. defconfig: defconfig name.
  590. Returns:
  591. Return a tuple of (updated flag, log string).
  592. The "updated flag" is True if the .config was updated, False
  593. otherwise. The "log string" shows what happend to the .config.
  594. """
  595. results = []
  596. updated = False
  597. suspicious = False
  598. rm_files = [self.config_autoconf, self.autoconf]
  599. if self.args.spl:
  600. if os.path.exists(self.spl_autoconf):
  601. autoconf_path = self.spl_autoconf
  602. rm_files.append(self.spl_autoconf)
  603. else:
  604. for f in rm_files:
  605. os.remove(f)
  606. return (updated, suspicious,
  607. color_text(self.args.color, COLOR_BROWN,
  608. "SPL is not enabled. Skipped.") + '\n')
  609. else:
  610. autoconf_path = self.autoconf
  611. dotconfig_lines = read_file(self.dotconfig)
  612. autoconf_lines = read_file(autoconf_path)
  613. for config in self.configs:
  614. result = self.parse_one_config(config, dotconfig_lines,
  615. autoconf_lines)
  616. results.append(result)
  617. log = ''
  618. for (action, value) in results:
  619. if action == ACTION_MOVE:
  620. actlog = "Move '%s'" % value
  621. log_color = COLOR_LIGHT_GREEN
  622. elif action == ACTION_NO_ENTRY:
  623. actlog = '%s is not defined in Kconfig. Do nothing.' % value
  624. log_color = COLOR_LIGHT_BLUE
  625. elif action == ACTION_NO_ENTRY_WARN:
  626. actlog = '%s is not defined in Kconfig (suspicious). Do nothing.' % value
  627. log_color = COLOR_YELLOW
  628. suspicious = True
  629. elif action == ACTION_NO_CHANGE:
  630. actlog = "'%s' is the same as the define in Kconfig. Do nothing." \
  631. % value
  632. log_color = COLOR_LIGHT_PURPLE
  633. elif action == ACTION_SPL_NOT_EXIST:
  634. actlog = 'SPL is not enabled for this defconfig. Skip.'
  635. log_color = COLOR_PURPLE
  636. else:
  637. sys.exit('Internal Error. This should not happen.')
  638. log += color_text(self.args.color, log_color, actlog) + '\n'
  639. with open(self.dotconfig, 'a', encoding='utf-8') as out:
  640. for (action, value) in results:
  641. if action == ACTION_MOVE:
  642. out.write(value + '\n')
  643. updated = True
  644. self.results = results
  645. for f in rm_files:
  646. os.remove(f)
  647. return (updated, suspicious, log)
  648. def check_defconfig(self):
  649. """Check the defconfig after savedefconfig
  650. Returns:
  651. Return additional log if moved CONFIGs were removed again by
  652. 'make savedefconfig'.
  653. """
  654. log = ''
  655. defconfig_lines = read_file(self.defconfig)
  656. for (action, value) in self.results:
  657. if action != ACTION_MOVE:
  658. continue
  659. if not value in defconfig_lines:
  660. log += color_text(self.args.color, COLOR_YELLOW,
  661. "'%s' was removed by savedefconfig.\n" %
  662. value)
  663. return log
  664. class DatabaseThread(threading.Thread):
  665. """This thread processes results from Slot threads.
  666. It collects the data in the master config directary. There is only one
  667. result thread, and this helps to serialise the build output.
  668. """
  669. def __init__(self, config_db, db_queue):
  670. """Set up a new result thread
  671. Args:
  672. builder: Builder which will be sent each result
  673. """
  674. threading.Thread.__init__(self)
  675. self.config_db = config_db
  676. self.db_queue= db_queue
  677. def run(self):
  678. """Called to start up the result thread.
  679. We collect the next result job and pass it on to the build.
  680. """
  681. while True:
  682. defconfig, configs = self.db_queue.get()
  683. self.config_db[defconfig] = configs
  684. self.db_queue.task_done()
  685. class Slot:
  686. """A slot to store a subprocess.
  687. Each instance of this class handles one subprocess.
  688. This class is useful to control multiple threads
  689. for faster processing.
  690. """
  691. def __init__(self, toolchains, configs, args, progress, devnull,
  692. make_cmd, reference_src_dir, db_queue):
  693. """Create a new process slot.
  694. Args:
  695. toolchains: Toolchains object containing toolchains.
  696. configs: A list of CONFIGs to move.
  697. args: Program arguments
  698. progress: A progress indicator.
  699. devnull: A file object of '/dev/null'.
  700. make_cmd: command name of GNU Make.
  701. reference_src_dir: Determine the true starting config state from this
  702. source tree.
  703. db_queue: output queue to write config info for the database
  704. """
  705. self.toolchains = toolchains
  706. self.args = args
  707. self.progress = progress
  708. self.build_dir = tempfile.mkdtemp()
  709. self.devnull = devnull
  710. self.make_cmd = (make_cmd, 'O=' + self.build_dir)
  711. self.reference_src_dir = reference_src_dir
  712. self.db_queue = db_queue
  713. self.parser = KconfigParser(configs, args, self.build_dir)
  714. self.state = STATE_IDLE
  715. self.failed_boards = set()
  716. self.suspicious_boards = set()
  717. def __del__(self):
  718. """Delete the working directory
  719. This function makes sure the temporary directory is cleaned away
  720. even if Python suddenly dies due to error. It should be done in here
  721. because it is guaranteed the destructor is always invoked when the
  722. instance of the class gets unreferenced.
  723. If the subprocess is still running, wait until it finishes.
  724. """
  725. if self.state != STATE_IDLE:
  726. while self.ps.poll() == None:
  727. pass
  728. shutil.rmtree(self.build_dir)
  729. def add(self, defconfig):
  730. """Assign a new subprocess for defconfig and add it to the slot.
  731. If the slot is vacant, create a new subprocess for processing the
  732. given defconfig and add it to the slot. Just returns False if
  733. the slot is occupied (i.e. the current subprocess is still running).
  734. Args:
  735. defconfig: defconfig name.
  736. Returns:
  737. Return True on success or False on failure
  738. """
  739. if self.state != STATE_IDLE:
  740. return False
  741. self.defconfig = defconfig
  742. self.log = ''
  743. self.current_src_dir = self.reference_src_dir
  744. self.do_defconfig()
  745. return True
  746. def poll(self):
  747. """Check the status of the subprocess and handle it as needed.
  748. Returns True if the slot is vacant (i.e. in idle state).
  749. If the configuration is successfully finished, assign a new
  750. subprocess to build include/autoconf.mk.
  751. If include/autoconf.mk is generated, invoke the parser to
  752. parse the .config and the include/autoconf.mk, moving
  753. config options to the .config as needed.
  754. If the .config was updated, run "make savedefconfig" to sync
  755. it, update the original defconfig, and then set the slot back
  756. to the idle state.
  757. Returns:
  758. Return True if the subprocess is terminated, False otherwise
  759. """
  760. if self.state == STATE_IDLE:
  761. return True
  762. if self.ps.poll() == None:
  763. return False
  764. if self.ps.poll() != 0:
  765. self.handle_error()
  766. elif self.state == STATE_DEFCONFIG:
  767. if self.reference_src_dir and not self.current_src_dir:
  768. self.do_savedefconfig()
  769. else:
  770. self.do_autoconf()
  771. elif self.state == STATE_AUTOCONF:
  772. if self.current_src_dir:
  773. self.current_src_dir = None
  774. self.do_defconfig()
  775. elif self.args.build_db:
  776. self.do_build_db()
  777. else:
  778. self.do_savedefconfig()
  779. elif self.state == STATE_SAVEDEFCONFIG:
  780. self.update_defconfig()
  781. else:
  782. sys.exit('Internal Error. This should not happen.')
  783. return True if self.state == STATE_IDLE else False
  784. def handle_error(self):
  785. """Handle error cases."""
  786. self.log += color_text(self.args.color, COLOR_LIGHT_RED,
  787. 'Failed to process.\n')
  788. if self.args.verbose:
  789. self.log += color_text(self.args.color, COLOR_LIGHT_CYAN,
  790. self.ps.stderr.read().decode())
  791. self.finish(False)
  792. def do_defconfig(self):
  793. """Run 'make <board>_defconfig' to create the .config file."""
  794. cmd = list(self.make_cmd)
  795. cmd.append(self.defconfig)
  796. self.ps = subprocess.Popen(cmd, stdout=self.devnull,
  797. stderr=subprocess.PIPE,
  798. cwd=self.current_src_dir)
  799. self.state = STATE_DEFCONFIG
  800. def do_autoconf(self):
  801. """Run 'make AUTO_CONF_PATH'."""
  802. arch = self.parser.get_arch()
  803. try:
  804. toolchain = self.toolchains.Select(arch)
  805. except ValueError:
  806. self.log += color_text(self.args.color, COLOR_YELLOW,
  807. "Tool chain for '%s' is missing. Do nothing.\n" % arch)
  808. self.finish(False)
  809. return
  810. env = toolchain.MakeEnvironment(False)
  811. cmd = list(self.make_cmd)
  812. cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
  813. cmd.append(AUTO_CONF_PATH)
  814. self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env,
  815. stderr=subprocess.PIPE,
  816. cwd=self.current_src_dir)
  817. self.state = STATE_AUTOCONF
  818. def do_build_db(self):
  819. """Add the board to the database"""
  820. configs = {}
  821. for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)):
  822. if line.startswith('CONFIG'):
  823. config, value = line.split('=', 1)
  824. configs[config] = value.rstrip()
  825. self.db_queue.put([self.defconfig, configs])
  826. self.finish(True)
  827. def do_savedefconfig(self):
  828. """Update the .config and run 'make savedefconfig'."""
  829. (updated, suspicious, log) = self.parser.update_dotconfig()
  830. if suspicious:
  831. self.suspicious_boards.add(self.defconfig)
  832. self.log += log
  833. if not self.args.force_sync and not updated:
  834. self.finish(True)
  835. return
  836. if updated:
  837. self.log += color_text(self.args.color, COLOR_LIGHT_GREEN,
  838. 'Syncing by savedefconfig...\n')
  839. else:
  840. self.log += 'Syncing by savedefconfig (forced by option)...\n'
  841. cmd = list(self.make_cmd)
  842. cmd.append('savedefconfig')
  843. self.ps = subprocess.Popen(cmd, stdout=self.devnull,
  844. stderr=subprocess.PIPE)
  845. self.state = STATE_SAVEDEFCONFIG
  846. def update_defconfig(self):
  847. """Update the input defconfig and go back to the idle state."""
  848. log = self.parser.check_defconfig()
  849. if log:
  850. self.suspicious_boards.add(self.defconfig)
  851. self.log += log
  852. orig_defconfig = os.path.join('configs', self.defconfig)
  853. new_defconfig = os.path.join(self.build_dir, 'defconfig')
  854. updated = not filecmp.cmp(orig_defconfig, new_defconfig)
  855. if updated:
  856. self.log += color_text(self.args.color, COLOR_LIGHT_BLUE,
  857. 'defconfig was updated.\n')
  858. if not self.args.dry_run and updated:
  859. shutil.move(new_defconfig, orig_defconfig)
  860. self.finish(True)
  861. def finish(self, success):
  862. """Display log along with progress and go to the idle state.
  863. Args:
  864. success: Should be True when the defconfig was processed
  865. successfully, or False when it fails.
  866. """
  867. # output at least 30 characters to hide the "* defconfigs out of *".
  868. log = self.defconfig.ljust(30) + '\n'
  869. log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
  870. # Some threads are running in parallel.
  871. # Print log atomically to not mix up logs from different threads.
  872. print(log, file=(sys.stdout if success else sys.stderr))
  873. if not success:
  874. if self.args.exit_on_error:
  875. sys.exit('Exit on error.')
  876. # If --exit-on-error flag is not set, skip this board and continue.
  877. # Record the failed board.
  878. self.failed_boards.add(self.defconfig)
  879. self.progress.inc()
  880. self.progress.show()
  881. self.state = STATE_IDLE
  882. def get_failed_boards(self):
  883. """Returns a set of failed boards (defconfigs) in this slot.
  884. """
  885. return self.failed_boards
  886. def get_suspicious_boards(self):
  887. """Returns a set of boards (defconfigs) with possible misconversion.
  888. """
  889. return self.suspicious_boards - self.failed_boards
  890. class Slots:
  891. """Controller of the array of subprocess slots."""
  892. def __init__(self, toolchains, configs, args, progress,
  893. reference_src_dir, db_queue):
  894. """Create a new slots controller.
  895. Args:
  896. toolchains: Toolchains object containing toolchains.
  897. configs: A list of CONFIGs to move.
  898. args: Program arguments
  899. progress: A progress indicator.
  900. reference_src_dir: Determine the true starting config state from this
  901. source tree.
  902. db_queue: output queue to write config info for the database
  903. """
  904. self.args = args
  905. self.slots = []
  906. devnull = subprocess.DEVNULL
  907. make_cmd = get_make_cmd()
  908. for i in range(args.jobs):
  909. self.slots.append(Slot(toolchains, configs, args, progress,
  910. devnull, make_cmd, reference_src_dir,
  911. db_queue))
  912. def add(self, defconfig):
  913. """Add a new subprocess if a vacant slot is found.
  914. Args:
  915. defconfig: defconfig name to be put into.
  916. Returns:
  917. Return True on success or False on failure
  918. """
  919. for slot in self.slots:
  920. if slot.add(defconfig):
  921. return True
  922. return False
  923. def available(self):
  924. """Check if there is a vacant slot.
  925. Returns:
  926. Return True if at lease one vacant slot is found, False otherwise.
  927. """
  928. for slot in self.slots:
  929. if slot.poll():
  930. return True
  931. return False
  932. def empty(self):
  933. """Check if all slots are vacant.
  934. Returns:
  935. Return True if all the slots are vacant, False otherwise.
  936. """
  937. ret = True
  938. for slot in self.slots:
  939. if not slot.poll():
  940. ret = False
  941. return ret
  942. def show_failed_boards(self):
  943. """Display all of the failed boards (defconfigs)."""
  944. boards = set()
  945. output_file = 'moveconfig.failed'
  946. for slot in self.slots:
  947. boards |= slot.get_failed_boards()
  948. if boards:
  949. boards = '\n'.join(boards) + '\n'
  950. msg = 'The following boards were not processed due to error:\n'
  951. msg += boards
  952. msg += '(the list has been saved in %s)\n' % output_file
  953. print(color_text(self.args.color, COLOR_LIGHT_RED,
  954. msg), file=sys.stderr)
  955. write_file(output_file, boards)
  956. def show_suspicious_boards(self):
  957. """Display all boards (defconfigs) with possible misconversion."""
  958. boards = set()
  959. output_file = 'moveconfig.suspicious'
  960. for slot in self.slots:
  961. boards |= slot.get_suspicious_boards()
  962. if boards:
  963. boards = '\n'.join(boards) + '\n'
  964. msg = 'The following boards might have been converted incorrectly.\n'
  965. msg += 'It is highly recommended to check them manually:\n'
  966. msg += boards
  967. msg += '(the list has been saved in %s)\n' % output_file
  968. print(color_text(self.args.color, COLOR_YELLOW,
  969. msg), file=sys.stderr)
  970. write_file(output_file, boards)
  971. class ReferenceSource:
  972. """Reference source against which original configs should be parsed."""
  973. def __init__(self, commit):
  974. """Create a reference source directory based on a specified commit.
  975. Args:
  976. commit: commit to git-clone
  977. """
  978. self.src_dir = tempfile.mkdtemp()
  979. print('Cloning git repo to a separate work directory...')
  980. subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
  981. cwd=self.src_dir)
  982. print("Checkout '%s' to build the original autoconf.mk." % \
  983. subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip())
  984. subprocess.check_output(['git', 'checkout', commit],
  985. stderr=subprocess.STDOUT, cwd=self.src_dir)
  986. def __del__(self):
  987. """Delete the reference source directory
  988. This function makes sure the temporary directory is cleaned away
  989. even if Python suddenly dies due to error. It should be done in here
  990. because it is guaranteed the destructor is always invoked when the
  991. instance of the class gets unreferenced.
  992. """
  993. shutil.rmtree(self.src_dir)
  994. def get_dir(self):
  995. """Return the absolute path to the reference source directory."""
  996. return self.src_dir
  997. def move_config(toolchains, configs, args, db_queue):
  998. """Move config options to defconfig files.
  999. Args:
  1000. configs: A list of CONFIGs to move.
  1001. args: Program arguments
  1002. """
  1003. if len(configs) == 0:
  1004. if args.force_sync:
  1005. print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ')
  1006. elif args.build_db:
  1007. print('Building %s database' % CONFIG_DATABASE)
  1008. else:
  1009. print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ')
  1010. else:
  1011. print('Move ' + ', '.join(configs), end=' ')
  1012. print('(jobs: %d)\n' % args.jobs)
  1013. if args.git_ref:
  1014. reference_src = ReferenceSource(args.git_ref)
  1015. reference_src_dir = reference_src.get_dir()
  1016. else:
  1017. reference_src_dir = None
  1018. if args.defconfigs:
  1019. defconfigs = get_matched_defconfigs(args.defconfigs)
  1020. else:
  1021. defconfigs = get_all_defconfigs()
  1022. progress = Progress(len(defconfigs))
  1023. slots = Slots(toolchains, configs, args, progress, reference_src_dir,
  1024. db_queue)
  1025. # Main loop to process defconfig files:
  1026. # Add a new subprocess into a vacant slot.
  1027. # Sleep if there is no available slot.
  1028. for defconfig in defconfigs:
  1029. while not slots.add(defconfig):
  1030. while not slots.available():
  1031. # No available slot: sleep for a while
  1032. time.sleep(SLEEP_TIME)
  1033. # wait until all the subprocesses finish
  1034. while not slots.empty():
  1035. time.sleep(SLEEP_TIME)
  1036. print('')
  1037. slots.show_failed_boards()
  1038. slots.show_suspicious_boards()
  1039. def find_kconfig_rules(kconf, config, imply_config):
  1040. """Check whether a config has a 'select' or 'imply' keyword
  1041. Args:
  1042. kconf: Kconfiglib.Kconfig object
  1043. config: Name of config to check (without CONFIG_ prefix)
  1044. imply_config: Implying config (without CONFIG_ prefix) which may or
  1045. may not have an 'imply' for 'config')
  1046. Returns:
  1047. Symbol object for 'config' if found, else None
  1048. """
  1049. sym = kconf.syms.get(imply_config)
  1050. if sym:
  1051. for sel, cond in (sym.selects + sym.implies):
  1052. if sel.name == config:
  1053. return sym
  1054. return None
  1055. def check_imply_rule(kconf, config, imply_config):
  1056. """Check if we can add an 'imply' option
  1057. This finds imply_config in the Kconfig and looks to see if it is possible
  1058. to add an 'imply' for 'config' to that part of the Kconfig.
  1059. Args:
  1060. kconf: Kconfiglib.Kconfig object
  1061. config: Name of config to check (without CONFIG_ prefix)
  1062. imply_config: Implying config (without CONFIG_ prefix) which may or
  1063. may not have an 'imply' for 'config')
  1064. Returns:
  1065. tuple:
  1066. filename of Kconfig file containing imply_config, or None if none
  1067. line number within the Kconfig file, or 0 if none
  1068. message indicating the result
  1069. """
  1070. sym = kconf.syms.get(imply_config)
  1071. if not sym:
  1072. return 'cannot find sym'
  1073. nodes = sym.nodes
  1074. if len(nodes) != 1:
  1075. return '%d locations' % len(nodes)
  1076. node = nodes[0]
  1077. fname, linenum = node.filename, node.linenr
  1078. cwd = os.getcwd()
  1079. if cwd and fname.startswith(cwd):
  1080. fname = fname[len(cwd) + 1:]
  1081. file_line = ' at %s:%d' % (fname, linenum)
  1082. data = read_file(fname)
  1083. if data[linenum - 1] != 'config %s' % imply_config:
  1084. return None, 0, 'bad sym format %s%s' % (data[linenum], file_line)
  1085. return fname, linenum, 'adding%s' % file_line
  1086. def add_imply_rule(config, fname, linenum):
  1087. """Add a new 'imply' option to a Kconfig
  1088. Args:
  1089. config: config option to add an imply for (without CONFIG_ prefix)
  1090. fname: Kconfig filename to update
  1091. linenum: Line number to place the 'imply' before
  1092. Returns:
  1093. Message indicating the result
  1094. """
  1095. file_line = ' at %s:%d' % (fname, linenum)
  1096. data = read_file(fname)
  1097. linenum -= 1
  1098. for offset, line in enumerate(data[linenum:]):
  1099. if line.strip().startswith('help') or not line:
  1100. data.insert(linenum + offset, '\timply %s' % config)
  1101. write_file(fname, data)
  1102. return 'added%s' % file_line
  1103. return 'could not insert%s'
  1104. (IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = (
  1105. 1, 2, 4, 8)
  1106. IMPLY_FLAGS = {
  1107. 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'],
  1108. 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'],
  1109. 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'],
  1110. 'non-arch-board': [
  1111. IMPLY_NON_ARCH_BOARD,
  1112. 'Allow Kconfig options outside arch/ and /board/ to imply'],
  1113. }
  1114. def read_database():
  1115. """Read in the config database
  1116. Returns:
  1117. tuple:
  1118. set of all config options seen (each a str)
  1119. set of all defconfigs seen (each a str)
  1120. dict of configs for each defconfig:
  1121. key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig"
  1122. value: dict:
  1123. key: CONFIG option
  1124. value: Value of option
  1125. dict of defconfigs for each config:
  1126. key: CONFIG option
  1127. value: set of boards using that option
  1128. """
  1129. configs = {}
  1130. # key is defconfig name, value is dict of (CONFIG_xxx, value)
  1131. config_db = {}
  1132. # Set of all config options we have seen
  1133. all_configs = set()
  1134. # Set of all defconfigs we have seen
  1135. all_defconfigs = set()
  1136. defconfig_db = collections.defaultdict(set)
  1137. for line in read_file(CONFIG_DATABASE):
  1138. line = line.rstrip()
  1139. if not line: # Separator between defconfigs
  1140. config_db[defconfig] = configs
  1141. all_defconfigs.add(defconfig)
  1142. configs = {}
  1143. elif line[0] == ' ': # CONFIG line
  1144. config, value = line.strip().split('=', 1)
  1145. configs[config] = value
  1146. defconfig_db[config].add(defconfig)
  1147. all_configs.add(config)
  1148. else: # New defconfig
  1149. defconfig = line
  1150. return all_configs, all_defconfigs, config_db, defconfig_db
  1151. def do_imply_config(config_list, add_imply, imply_flags, skip_added,
  1152. check_kconfig=True, find_superset=False):
  1153. """Find CONFIG options which imply those in the list
  1154. Some CONFIG options can be implied by others and this can help to reduce
  1155. the size of the defconfig files. For example, CONFIG_X86 implies
  1156. CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and
  1157. all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to
  1158. each of the x86 defconfig files.
  1159. This function uses the moveconfig database to find such options. It
  1160. displays a list of things that could possibly imply those in the list.
  1161. The algorithm ignores any that start with CONFIG_TARGET since these
  1162. typically refer to only a few defconfigs (often one). It also does not
  1163. display a config with less than 5 defconfigs.
  1164. The algorithm works using sets. For each target config in config_list:
  1165. - Get the set 'defconfigs' which use that target config
  1166. - For each config (from a list of all configs):
  1167. - Get the set 'imply_defconfig' of defconfigs which use that config
  1168. -
  1169. - If imply_defconfigs contains anything not in defconfigs then
  1170. this config does not imply the target config
  1171. Params:
  1172. config_list: List of CONFIG options to check (each a string)
  1173. add_imply: Automatically add an 'imply' for each config.
  1174. imply_flags: Flags which control which implying configs are allowed
  1175. (IMPLY_...)
  1176. skip_added: Don't show options which already have an imply added.
  1177. check_kconfig: Check if implied symbols already have an 'imply' or
  1178. 'select' for the target config, and show this information if so.
  1179. find_superset: True to look for configs which are a superset of those
  1180. already found. So for example if CONFIG_EXYNOS5 implies an option,
  1181. but CONFIG_EXYNOS covers a larger set of defconfigs and also
  1182. implies that option, this will drop the former in favour of the
  1183. latter. In practice this option has not proved very used.
  1184. Note the terminoloy:
  1185. config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM')
  1186. defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig')
  1187. """
  1188. kconf = KconfigScanner().conf if check_kconfig else None
  1189. if add_imply and add_imply != 'all':
  1190. add_imply = add_imply.split(',')
  1191. all_configs, all_defconfigs, config_db, defconfig_db = read_database()
  1192. # Work through each target config option in turn, independently
  1193. for config in config_list:
  1194. defconfigs = defconfig_db.get(config)
  1195. if not defconfigs:
  1196. print('%s not found in any defconfig' % config)
  1197. continue
  1198. # Get the set of defconfigs without this one (since a config cannot
  1199. # imply itself)
  1200. non_defconfigs = all_defconfigs - defconfigs
  1201. num_defconfigs = len(defconfigs)
  1202. print('%s found in %d/%d defconfigs' % (config, num_defconfigs,
  1203. len(all_configs)))
  1204. # This will hold the results: key=config, value=defconfigs containing it
  1205. imply_configs = {}
  1206. rest_configs = all_configs - set([config])
  1207. # Look at every possible config, except the target one
  1208. for imply_config in rest_configs:
  1209. if 'ERRATUM' in imply_config:
  1210. continue
  1211. if not imply_flags & IMPLY_CMD:
  1212. if 'CONFIG_CMD' in imply_config:
  1213. continue
  1214. if not imply_flags & IMPLY_TARGET:
  1215. if 'CONFIG_TARGET' in imply_config:
  1216. continue
  1217. # Find set of defconfigs that have this config
  1218. imply_defconfig = defconfig_db[imply_config]
  1219. # Get the intersection of this with defconfigs containing the
  1220. # target config
  1221. common_defconfigs = imply_defconfig & defconfigs
  1222. # Get the set of defconfigs containing this config which DO NOT
  1223. # also contain the taret config. If this set is non-empty it means
  1224. # that this config affects other defconfigs as well as (possibly)
  1225. # the ones affected by the target config. This means it implies
  1226. # things we don't want to imply.
  1227. not_common_defconfigs = imply_defconfig & non_defconfigs
  1228. if not_common_defconfigs:
  1229. continue
  1230. # If there are common defconfigs, imply_config may be useful
  1231. if common_defconfigs:
  1232. skip = False
  1233. if find_superset:
  1234. for prev in list(imply_configs.keys()):
  1235. prev_count = len(imply_configs[prev])
  1236. count = len(common_defconfigs)
  1237. if (prev_count > count and
  1238. (imply_configs[prev] & common_defconfigs ==
  1239. common_defconfigs)):
  1240. # skip imply_config because prev is a superset
  1241. skip = True
  1242. break
  1243. elif count > prev_count:
  1244. # delete prev because imply_config is a superset
  1245. del imply_configs[prev]
  1246. if not skip:
  1247. imply_configs[imply_config] = common_defconfigs
  1248. # Now we have a dict imply_configs of configs which imply each config
  1249. # The value of each dict item is the set of defconfigs containing that
  1250. # config. Rank them so that we print the configs that imply the largest
  1251. # number of defconfigs first.
  1252. ranked_iconfigs = sorted(imply_configs,
  1253. key=lambda k: len(imply_configs[k]), reverse=True)
  1254. kconfig_info = ''
  1255. cwd = os.getcwd()
  1256. add_list = collections.defaultdict(list)
  1257. for iconfig in ranked_iconfigs:
  1258. num_common = len(imply_configs[iconfig])
  1259. # Don't bother if there are less than 5 defconfigs affected.
  1260. if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5):
  1261. continue
  1262. missing = defconfigs - imply_configs[iconfig]
  1263. missing_str = ', '.join(missing) if missing else 'all'
  1264. missing_str = ''
  1265. show = True
  1266. if kconf:
  1267. sym = find_kconfig_rules(kconf, config[CONFIG_LEN:],
  1268. iconfig[CONFIG_LEN:])
  1269. kconfig_info = ''
  1270. if sym:
  1271. nodes = sym.nodes
  1272. if len(nodes) == 1:
  1273. fname, linenum = nodes[0].filename, nodes[0].linenr
  1274. if cwd and fname.startswith(cwd):
  1275. fname = fname[len(cwd) + 1:]
  1276. kconfig_info = '%s:%d' % (fname, linenum)
  1277. if skip_added:
  1278. show = False
  1279. else:
  1280. sym = kconf.syms.get(iconfig[CONFIG_LEN:])
  1281. fname = ''
  1282. if sym:
  1283. nodes = sym.nodes
  1284. if len(nodes) == 1:
  1285. fname, linenum = nodes[0].filename, nodes[0].linenr
  1286. if cwd and fname.startswith(cwd):
  1287. fname = fname[len(cwd) + 1:]
  1288. in_arch_board = not sym or (fname.startswith('arch') or
  1289. fname.startswith('board'))
  1290. if (not in_arch_board and
  1291. not imply_flags & IMPLY_NON_ARCH_BOARD):
  1292. continue
  1293. if add_imply and (add_imply == 'all' or
  1294. iconfig in add_imply):
  1295. fname, linenum, kconfig_info = (check_imply_rule(kconf,
  1296. config[CONFIG_LEN:], iconfig[CONFIG_LEN:]))
  1297. if fname:
  1298. add_list[fname].append(linenum)
  1299. if show and kconfig_info != 'skip':
  1300. print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30),
  1301. kconfig_info, missing_str))
  1302. # Having collected a list of things to add, now we add them. We process
  1303. # each file from the largest line number to the smallest so that
  1304. # earlier additions do not affect our line numbers. E.g. if we added an
  1305. # imply at line 20 it would change the position of each line after
  1306. # that.
  1307. for fname, linenums in add_list.items():
  1308. for linenum in sorted(linenums, reverse=True):
  1309. add_imply_rule(config[CONFIG_LEN:], fname, linenum)
  1310. def do_find_config(config_list):
  1311. """Find boards with a given combination of CONFIGs
  1312. Params:
  1313. config_list: List of CONFIG options to check (each a string consisting
  1314. of a config option, with or without a CONFIG_ prefix. If an option
  1315. is preceded by a tilde (~) then it must be false, otherwise it must
  1316. be true)
  1317. """
  1318. all_configs, all_defconfigs, config_db, defconfig_db = read_database()
  1319. # Get the whitelist
  1320. adhoc_configs = set(read_file('scripts/config_whitelist.txt'))
  1321. # Start with all defconfigs
  1322. out = all_defconfigs
  1323. # Work through each config in turn
  1324. adhoc = []
  1325. for item in config_list:
  1326. # Get the real config name and whether we want this config or not
  1327. cfg = item
  1328. want = True
  1329. if cfg[0] == '~':
  1330. want = False
  1331. cfg = cfg[1:]
  1332. if cfg in adhoc_configs:
  1333. adhoc.append(cfg)
  1334. continue
  1335. # Search everything that is still in the running. If it has a config
  1336. # that we want, or doesn't have one that we don't, add it into the
  1337. # running for the next stage
  1338. in_list = out
  1339. out = set()
  1340. for defc in in_list:
  1341. has_cfg = cfg in config_db[defc]
  1342. if has_cfg == want:
  1343. out.add(defc)
  1344. if adhoc:
  1345. print(f"Error: Not in Kconfig: %s" % ' '.join(adhoc))
  1346. else:
  1347. print(f'{len(out)} matches')
  1348. print(' '.join(out))
  1349. def prefix_config(cfg):
  1350. """Prefix a config with CONFIG_ if needed
  1351. This handles ~ operator, which indicates that the CONFIG should be disabled
  1352. >>> prefix_config('FRED')
  1353. 'CONFIG_FRED'
  1354. >>> prefix_config('CONFIG_FRED')
  1355. 'CONFIG_FRED'
  1356. >>> prefix_config('~FRED')
  1357. '~CONFIG_FRED'
  1358. >>> prefix_config('~CONFIG_FRED')
  1359. '~CONFIG_FRED'
  1360. >>> prefix_config('A123')
  1361. 'CONFIG_A123'
  1362. """
  1363. op = ''
  1364. if cfg[0] == '~':
  1365. op = cfg[0]
  1366. cfg = cfg[1:]
  1367. if not cfg.startswith('CONFIG_'):
  1368. cfg = 'CONFIG_' + cfg
  1369. return op + cfg
  1370. def main():
  1371. try:
  1372. cpu_count = multiprocessing.cpu_count()
  1373. except NotImplementedError:
  1374. cpu_count = 1
  1375. epilog = '''Move config options from headers to defconfig files. See
  1376. doc/develop/moveconfig.rst for documentation.'''
  1377. parser = ArgumentParser(epilog=epilog)
  1378. # Add arguments here
  1379. parser.add_argument('-a', '--add-imply', type=str, default='',
  1380. help='comma-separated list of CONFIG options to add '
  1381. "an 'imply' statement to for the CONFIG in -i")
  1382. parser.add_argument('-A', '--skip-added', action='store_true', default=False,
  1383. help="don't show options which are already marked as "
  1384. 'implying others')
  1385. parser.add_argument('-b', '--build-db', action='store_true', default=False,
  1386. help='build a CONFIG database')
  1387. parser.add_argument('-c', '--color', action='store_true', default=False,
  1388. help='display the log in color')
  1389. parser.add_argument('-C', '--commit', action='store_true', default=False,
  1390. help='Create a git commit for the operation')
  1391. parser.add_argument('-d', '--defconfigs', type=str,
  1392. help='a file containing a list of defconfigs to move, '
  1393. "one per line (for example 'snow_defconfig') "
  1394. "or '-' to read from stdin")
  1395. parser.add_argument('-e', '--exit-on-error', action='store_true',
  1396. default=False,
  1397. help='exit immediately on any error')
  1398. parser.add_argument('-f', '--find', action='store_true', default=False,
  1399. help='Find boards with a given config combination')
  1400. parser.add_argument('-H', '--headers-only', dest='cleanup_headers_only',
  1401. action='store_true', default=False,
  1402. help='only cleanup the headers')
  1403. parser.add_argument('-i', '--imply', action='store_true', default=False,
  1404. help='find options which imply others')
  1405. parser.add_argument('-I', '--imply-flags', type=str, default='',
  1406. help="control the -i option ('help' for help")
  1407. parser.add_argument('-j', '--jobs', type=int, default=cpu_count,
  1408. help='the number of jobs to run simultaneously')
  1409. parser.add_argument('-n', '--dry-run', action='store_true', default=False,
  1410. help='perform a trial run (show log with no changes)')
  1411. parser.add_argument('-r', '--git-ref', type=str,
  1412. help='the git ref to clone for building the autoconf.mk')
  1413. parser.add_argument('-s', '--force-sync', action='store_true', default=False,
  1414. help='force sync by savedefconfig')
  1415. parser.add_argument('-S', '--spl', action='store_true', default=False,
  1416. help='parse config options defined for SPL build')
  1417. parser.add_argument('-t', '--test', action='store_true', default=False,
  1418. help='run unit tests')
  1419. parser.add_argument('-y', '--yes', action='store_true', default=False,
  1420. help="respond 'yes' to any prompts")
  1421. parser.add_argument('-v', '--verbose', action='store_true', default=False,
  1422. help='show any build errors as boards are built')
  1423. parser.add_argument('configs', nargs='*')
  1424. args = parser.parse_args()
  1425. configs = args.configs
  1426. if args.test:
  1427. sys.argv = [sys.argv[0]]
  1428. fail, count = doctest.testmod()
  1429. if fail:
  1430. return 1
  1431. unittest.main()
  1432. if not any((len(configs), args.force_sync, args.build_db, args.imply,
  1433. args.find)):
  1434. parser.print_usage()
  1435. sys.exit(1)
  1436. # prefix the option name with CONFIG_ if missing
  1437. configs = [prefix_config(cfg) for cfg in configs]
  1438. check_top_directory()
  1439. if args.imply:
  1440. imply_flags = 0
  1441. if args.imply_flags == 'all':
  1442. imply_flags = -1
  1443. elif args.imply_flags:
  1444. for flag in args.imply_flags.split(','):
  1445. bad = flag not in IMPLY_FLAGS
  1446. if bad:
  1447. print("Invalid flag '%s'" % flag)
  1448. if flag == 'help' or bad:
  1449. print("Imply flags: (separate with ',')")
  1450. for name, info in IMPLY_FLAGS.items():
  1451. print(' %-15s: %s' % (name, info[1]))
  1452. parser.print_usage()
  1453. sys.exit(1)
  1454. imply_flags |= IMPLY_FLAGS[flag][0]
  1455. do_imply_config(configs, args.add_imply, imply_flags, args.skip_added)
  1456. return
  1457. if args.find:
  1458. do_find_config(configs)
  1459. return
  1460. config_db = {}
  1461. db_queue = queue.Queue()
  1462. t = DatabaseThread(config_db, db_queue)
  1463. t.setDaemon(True)
  1464. t.start()
  1465. if not args.cleanup_headers_only:
  1466. check_clean_directory()
  1467. bsettings.Setup('')
  1468. toolchains = toolchain.Toolchains()
  1469. toolchains.GetSettings()
  1470. toolchains.Scan(verbose=False)
  1471. move_config(toolchains, configs, args, db_queue)
  1472. db_queue.join()
  1473. if configs:
  1474. cleanup_headers(configs, args)
  1475. cleanup_extra_options(configs, args)
  1476. cleanup_whitelist(configs, args)
  1477. cleanup_readme(configs, args)
  1478. if args.commit:
  1479. subprocess.call(['git', 'add', '-u'])
  1480. if configs:
  1481. msg = 'Convert %s %sto Kconfig' % (configs[0],
  1482. 'et al ' if len(configs) > 1 else '')
  1483. msg += ('\n\nThis converts the following to Kconfig:\n %s\n' %
  1484. '\n '.join(configs))
  1485. else:
  1486. msg = 'configs: Resync with savedefconfig'
  1487. msg += '\n\nRsync all defconfig files using moveconfig.py'
  1488. subprocess.call(['git', 'commit', '-s', '-m', msg])
  1489. if args.build_db:
  1490. with open(CONFIG_DATABASE, 'w', encoding='utf-8') as fd:
  1491. for defconfig, configs in config_db.items():
  1492. fd.write('%s\n' % defconfig)
  1493. for config in sorted(configs.keys()):
  1494. fd.write(' %s=%s\n' % (config, configs[config]))
  1495. fd.write('\n')
  1496. if __name__ == '__main__':
  1497. sys.exit(main())