moveconfig.py 58 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676
  1. #!/usr/bin/env python3
  2. # SPDX-License-Identifier: GPL-2.0+
  3. #
  4. # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
  5. #
  6. """
  7. Move config options from headers to defconfig files.
  8. See doc/develop/moveconfig.rst for documentation.
  9. """
  10. import asteval
  11. import collections
  12. import copy
  13. import difflib
  14. import filecmp
  15. import fnmatch
  16. import glob
  17. import multiprocessing
  18. import optparse
  19. import os
  20. import queue
  21. import re
  22. import shutil
  23. import subprocess
  24. import sys
  25. import tempfile
  26. import threading
  27. import time
  28. from buildman import bsettings
  29. from buildman import kconfiglib
  30. from buildman import toolchain
  31. SHOW_GNU_MAKE = 'scripts/show-gnu-make'
  32. SLEEP_TIME=0.03
  33. STATE_IDLE = 0
  34. STATE_DEFCONFIG = 1
  35. STATE_AUTOCONF = 2
  36. STATE_SAVEDEFCONFIG = 3
  37. ACTION_MOVE = 0
  38. ACTION_NO_ENTRY = 1
  39. ACTION_NO_ENTRY_WARN = 2
  40. ACTION_NO_CHANGE = 3
  41. COLOR_BLACK = '0;30'
  42. COLOR_RED = '0;31'
  43. COLOR_GREEN = '0;32'
  44. COLOR_BROWN = '0;33'
  45. COLOR_BLUE = '0;34'
  46. COLOR_PURPLE = '0;35'
  47. COLOR_CYAN = '0;36'
  48. COLOR_LIGHT_GRAY = '0;37'
  49. COLOR_DARK_GRAY = '1;30'
  50. COLOR_LIGHT_RED = '1;31'
  51. COLOR_LIGHT_GREEN = '1;32'
  52. COLOR_YELLOW = '1;33'
  53. COLOR_LIGHT_BLUE = '1;34'
  54. COLOR_LIGHT_PURPLE = '1;35'
  55. COLOR_LIGHT_CYAN = '1;36'
  56. COLOR_WHITE = '1;37'
  57. AUTO_CONF_PATH = 'include/config/auto.conf'
  58. CONFIG_DATABASE = 'moveconfig.db'
  59. CONFIG_LEN = len('CONFIG_')
  60. SIZES = {
  61. "SZ_1": 0x00000001, "SZ_2": 0x00000002,
  62. "SZ_4": 0x00000004, "SZ_8": 0x00000008,
  63. "SZ_16": 0x00000010, "SZ_32": 0x00000020,
  64. "SZ_64": 0x00000040, "SZ_128": 0x00000080,
  65. "SZ_256": 0x00000100, "SZ_512": 0x00000200,
  66. "SZ_1K": 0x00000400, "SZ_2K": 0x00000800,
  67. "SZ_4K": 0x00001000, "SZ_8K": 0x00002000,
  68. "SZ_16K": 0x00004000, "SZ_32K": 0x00008000,
  69. "SZ_64K": 0x00010000, "SZ_128K": 0x00020000,
  70. "SZ_256K": 0x00040000, "SZ_512K": 0x00080000,
  71. "SZ_1M": 0x00100000, "SZ_2M": 0x00200000,
  72. "SZ_4M": 0x00400000, "SZ_8M": 0x00800000,
  73. "SZ_16M": 0x01000000, "SZ_32M": 0x02000000,
  74. "SZ_64M": 0x04000000, "SZ_128M": 0x08000000,
  75. "SZ_256M": 0x10000000, "SZ_512M": 0x20000000,
  76. "SZ_1G": 0x40000000, "SZ_2G": 0x80000000,
  77. "SZ_4G": 0x100000000
  78. }
  79. ### helper functions ###
  80. def get_devnull():
  81. """Get the file object of '/dev/null' device."""
  82. try:
  83. devnull = subprocess.DEVNULL # py3k
  84. except AttributeError:
  85. devnull = open(os.devnull, 'wb')
  86. return devnull
  87. def check_top_directory():
  88. """Exit if we are not at the top of source directory."""
  89. for f in ('README', 'Licenses'):
  90. if not os.path.exists(f):
  91. sys.exit('Please run at the top of source directory.')
  92. def check_clean_directory():
  93. """Exit if the source tree is not clean."""
  94. for f in ('.config', 'include/config'):
  95. if os.path.exists(f):
  96. sys.exit("source tree is not clean, please run 'make mrproper'")
  97. def get_make_cmd():
  98. """Get the command name of GNU Make.
  99. U-Boot needs GNU Make for building, but the command name is not
  100. necessarily "make". (for example, "gmake" on FreeBSD).
  101. Returns the most appropriate command name on your system.
  102. """
  103. process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
  104. ret = process.communicate()
  105. if process.returncode:
  106. sys.exit('GNU Make not found')
  107. return ret[0].rstrip()
  108. def get_matched_defconfig(line):
  109. """Get the defconfig files that match a pattern
  110. Args:
  111. line: Path or filename to match, e.g. 'configs/snow_defconfig' or
  112. 'k2*_defconfig'. If no directory is provided, 'configs/' is
  113. prepended
  114. Returns:
  115. a list of matching defconfig files
  116. """
  117. dirname = os.path.dirname(line)
  118. if dirname:
  119. pattern = line
  120. else:
  121. pattern = os.path.join('configs', line)
  122. return glob.glob(pattern) + glob.glob(pattern + '_defconfig')
  123. def get_matched_defconfigs(defconfigs_file):
  124. """Get all the defconfig files that match the patterns in a file.
  125. Args:
  126. defconfigs_file: File containing a list of defconfigs to process, or
  127. '-' to read the list from stdin
  128. Returns:
  129. A list of paths to defconfig files, with no duplicates
  130. """
  131. defconfigs = []
  132. if defconfigs_file == '-':
  133. fd = sys.stdin
  134. defconfigs_file = 'stdin'
  135. else:
  136. fd = open(defconfigs_file)
  137. for i, line in enumerate(fd):
  138. line = line.strip()
  139. if not line:
  140. continue # skip blank lines silently
  141. if ' ' in line:
  142. line = line.split(' ')[0] # handle 'git log' input
  143. matched = get_matched_defconfig(line)
  144. if not matched:
  145. print("warning: %s:%d: no defconfig matched '%s'" % \
  146. (defconfigs_file, i + 1, line), file=sys.stderr)
  147. defconfigs += matched
  148. # use set() to drop multiple matching
  149. return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ]
  150. def get_all_defconfigs():
  151. """Get all the defconfig files under the configs/ directory."""
  152. defconfigs = []
  153. for (dirpath, dirnames, filenames) in os.walk('configs'):
  154. dirpath = dirpath[len('configs') + 1:]
  155. for filename in fnmatch.filter(filenames, '*_defconfig'):
  156. defconfigs.append(os.path.join(dirpath, filename))
  157. return defconfigs
  158. def color_text(color_enabled, color, string):
  159. """Return colored string."""
  160. if color_enabled:
  161. # LF should not be surrounded by the escape sequence.
  162. # Otherwise, additional whitespace or line-feed might be printed.
  163. return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
  164. for s in string.split('\n') ])
  165. else:
  166. return string
  167. def show_diff(a, b, file_path, color_enabled):
  168. """Show unidified diff.
  169. Arguments:
  170. a: A list of lines (before)
  171. b: A list of lines (after)
  172. file_path: Path to the file
  173. color_enabled: Display the diff in color
  174. """
  175. diff = difflib.unified_diff(a, b,
  176. fromfile=os.path.join('a', file_path),
  177. tofile=os.path.join('b', file_path))
  178. for line in diff:
  179. if line[0] == '-' and line[1] != '-':
  180. print(color_text(color_enabled, COLOR_RED, line), end=' ')
  181. elif line[0] == '+' and line[1] != '+':
  182. print(color_text(color_enabled, COLOR_GREEN, line), end=' ')
  183. else:
  184. print(line, end=' ')
  185. def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre,
  186. extend_post):
  187. """Extend matched lines if desired patterns are found before/after already
  188. matched lines.
  189. Arguments:
  190. lines: A list of lines handled.
  191. matched: A list of line numbers that have been already matched.
  192. (will be updated by this function)
  193. pre_patterns: A list of regular expression that should be matched as
  194. preamble.
  195. post_patterns: A list of regular expression that should be matched as
  196. postamble.
  197. extend_pre: Add the line number of matched preamble to the matched list.
  198. extend_post: Add the line number of matched postamble to the matched list.
  199. """
  200. extended_matched = []
  201. j = matched[0]
  202. for i in matched:
  203. if i == 0 or i < j:
  204. continue
  205. j = i
  206. while j in matched:
  207. j += 1
  208. if j >= len(lines):
  209. break
  210. for p in pre_patterns:
  211. if p.search(lines[i - 1]):
  212. break
  213. else:
  214. # not matched
  215. continue
  216. for p in post_patterns:
  217. if p.search(lines[j]):
  218. break
  219. else:
  220. # not matched
  221. continue
  222. if extend_pre:
  223. extended_matched.append(i - 1)
  224. if extend_post:
  225. extended_matched.append(j)
  226. matched += extended_matched
  227. matched.sort()
  228. def confirm(options, prompt):
  229. if not options.yes:
  230. while True:
  231. choice = input('{} [y/n]: '.format(prompt))
  232. choice = choice.lower()
  233. print(choice)
  234. if choice == 'y' or choice == 'n':
  235. break
  236. if choice == 'n':
  237. return False
  238. return True
  239. def cleanup_empty_blocks(header_path, options):
  240. """Clean up empty conditional blocks
  241. Arguments:
  242. header_path: path to the cleaned file.
  243. options: option flags.
  244. """
  245. pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M)
  246. with open(header_path) as f:
  247. try:
  248. data = f.read()
  249. except UnicodeDecodeError as e:
  250. print("Failed on file %s': %s" % (header_path, e))
  251. return
  252. new_data = pattern.sub('\n', data)
  253. show_diff(data.splitlines(True), new_data.splitlines(True), header_path,
  254. options.color)
  255. if options.dry_run:
  256. return
  257. with open(header_path, 'w') as f:
  258. f.write(new_data)
  259. def cleanup_one_header(header_path, patterns, options):
  260. """Clean regex-matched lines away from a file.
  261. Arguments:
  262. header_path: path to the cleaned file.
  263. patterns: list of regex patterns. Any lines matching to these
  264. patterns are deleted.
  265. options: option flags.
  266. """
  267. with open(header_path) as f:
  268. try:
  269. lines = f.readlines()
  270. except UnicodeDecodeError as e:
  271. print("Failed on file %s': %s" % (header_path, e))
  272. return
  273. matched = []
  274. for i, line in enumerate(lines):
  275. if i - 1 in matched and lines[i - 1][-2:] == '\\\n':
  276. matched.append(i)
  277. continue
  278. for pattern in patterns:
  279. if pattern.search(line):
  280. matched.append(i)
  281. break
  282. if not matched:
  283. return
  284. # remove empty #ifdef ... #endif, successive blank lines
  285. pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef
  286. pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else
  287. pattern_endif = re.compile(r'#\s*endif\W') # #endif
  288. pattern_blank = re.compile(r'^\s*$') # empty line
  289. while True:
  290. old_matched = copy.copy(matched)
  291. extend_matched_lines(lines, matched, [pattern_if],
  292. [pattern_endif], True, True)
  293. extend_matched_lines(lines, matched, [pattern_elif],
  294. [pattern_elif, pattern_endif], True, False)
  295. extend_matched_lines(lines, matched, [pattern_if, pattern_elif],
  296. [pattern_blank], False, True)
  297. extend_matched_lines(lines, matched, [pattern_blank],
  298. [pattern_elif, pattern_endif], True, False)
  299. extend_matched_lines(lines, matched, [pattern_blank],
  300. [pattern_blank], True, False)
  301. if matched == old_matched:
  302. break
  303. tolines = copy.copy(lines)
  304. for i in reversed(matched):
  305. tolines.pop(i)
  306. show_diff(lines, tolines, header_path, options.color)
  307. if options.dry_run:
  308. return
  309. with open(header_path, 'w') as f:
  310. for line in tolines:
  311. f.write(line)
  312. def cleanup_headers(configs, options):
  313. """Delete config defines from board headers.
  314. Arguments:
  315. configs: A list of CONFIGs to remove.
  316. options: option flags.
  317. """
  318. if not confirm(options, 'Clean up headers?'):
  319. return
  320. patterns = []
  321. for config in configs:
  322. patterns.append(re.compile(r'#\s*define\s+%s\W' % config))
  323. patterns.append(re.compile(r'#\s*undef\s+%s\W' % config))
  324. for dir in 'include', 'arch', 'board':
  325. for (dirpath, dirnames, filenames) in os.walk(dir):
  326. if dirpath == os.path.join('include', 'generated'):
  327. continue
  328. for filename in filenames:
  329. if not filename.endswith(('~', '.dts', '.dtsi', '.bin',
  330. '.elf','.aml','.dat')):
  331. header_path = os.path.join(dirpath, filename)
  332. # This file contains UTF-16 data and no CONFIG symbols
  333. if header_path == 'include/video_font_data.h':
  334. continue
  335. cleanup_one_header(header_path, patterns, options)
  336. cleanup_empty_blocks(header_path, options)
  337. def cleanup_one_extra_option(defconfig_path, configs, options):
  338. """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file.
  339. Arguments:
  340. defconfig_path: path to the cleaned defconfig file.
  341. configs: A list of CONFIGs to remove.
  342. options: option flags.
  343. """
  344. start = 'CONFIG_SYS_EXTRA_OPTIONS="'
  345. end = '"\n'
  346. with open(defconfig_path) as f:
  347. lines = f.readlines()
  348. for i, line in enumerate(lines):
  349. if line.startswith(start) and line.endswith(end):
  350. break
  351. else:
  352. # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig
  353. return
  354. old_tokens = line[len(start):-len(end)].split(',')
  355. new_tokens = []
  356. for token in old_tokens:
  357. pos = token.find('=')
  358. if not (token[:pos] if pos >= 0 else token) in configs:
  359. new_tokens.append(token)
  360. if new_tokens == old_tokens:
  361. return
  362. tolines = copy.copy(lines)
  363. if new_tokens:
  364. tolines[i] = start + ','.join(new_tokens) + end
  365. else:
  366. tolines.pop(i)
  367. show_diff(lines, tolines, defconfig_path, options.color)
  368. if options.dry_run:
  369. return
  370. with open(defconfig_path, 'w') as f:
  371. for line in tolines:
  372. f.write(line)
  373. def cleanup_extra_options(configs, options):
  374. """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files.
  375. Arguments:
  376. configs: A list of CONFIGs to remove.
  377. options: option flags.
  378. """
  379. if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'):
  380. return
  381. configs = [ config[len('CONFIG_'):] for config in configs ]
  382. defconfigs = get_all_defconfigs()
  383. for defconfig in defconfigs:
  384. cleanup_one_extra_option(os.path.join('configs', defconfig), configs,
  385. options)
  386. def cleanup_whitelist(configs, options):
  387. """Delete config whitelist entries
  388. Arguments:
  389. configs: A list of CONFIGs to remove.
  390. options: option flags.
  391. """
  392. if not confirm(options, 'Clean up whitelist entries?'):
  393. return
  394. with open(os.path.join('scripts', 'config_whitelist.txt')) as f:
  395. lines = f.readlines()
  396. lines = [x for x in lines if x.strip() not in configs]
  397. with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f:
  398. f.write(''.join(lines))
  399. def find_matching(patterns, line):
  400. for pat in patterns:
  401. if pat.search(line):
  402. return True
  403. return False
  404. def cleanup_readme(configs, options):
  405. """Delete config description in README
  406. Arguments:
  407. configs: A list of CONFIGs to remove.
  408. options: option flags.
  409. """
  410. if not confirm(options, 'Clean up README?'):
  411. return
  412. patterns = []
  413. for config in configs:
  414. patterns.append(re.compile(r'^\s+%s' % config))
  415. with open('README') as f:
  416. lines = f.readlines()
  417. found = False
  418. newlines = []
  419. for line in lines:
  420. if not found:
  421. found = find_matching(patterns, line)
  422. if found:
  423. continue
  424. if found and re.search(r'^\s+CONFIG', line):
  425. found = False
  426. if not found:
  427. newlines.append(line)
  428. with open('README', 'w') as f:
  429. f.write(''.join(newlines))
  430. def try_expand(line):
  431. """If value looks like an expression, try expanding it
  432. Otherwise just return the existing value
  433. """
  434. if line.find('=') == -1:
  435. return line
  436. try:
  437. aeval = asteval.Interpreter( usersyms=SIZES, minimal=True )
  438. cfg, val = re.split("=", line)
  439. val= val.strip('\"')
  440. if re.search("[*+-/]|<<|SZ_+|\(([^\)]+)\)", val):
  441. newval = hex(aeval(val))
  442. print("\tExpanded expression %s to %s" % (val, newval))
  443. return cfg+'='+newval
  444. except:
  445. print("\tFailed to expand expression in %s" % line)
  446. return line
  447. ### classes ###
  448. class Progress:
  449. """Progress Indicator"""
  450. def __init__(self, total):
  451. """Create a new progress indicator.
  452. Arguments:
  453. total: A number of defconfig files to process.
  454. """
  455. self.current = 0
  456. self.total = total
  457. def inc(self):
  458. """Increment the number of processed defconfig files."""
  459. self.current += 1
  460. def show(self):
  461. """Display the progress."""
  462. print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ')
  463. sys.stdout.flush()
  464. class KconfigScanner:
  465. """Kconfig scanner."""
  466. def __init__(self):
  467. """Scan all the Kconfig files and create a Config object."""
  468. # Define environment variables referenced from Kconfig
  469. os.environ['srctree'] = os.getcwd()
  470. os.environ['UBOOTVERSION'] = 'dummy'
  471. os.environ['KCONFIG_OBJDIR'] = ''
  472. self.conf = kconfiglib.Kconfig()
  473. class KconfigParser:
  474. """A parser of .config and include/autoconf.mk."""
  475. re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
  476. re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
  477. def __init__(self, configs, options, build_dir):
  478. """Create a new parser.
  479. Arguments:
  480. configs: A list of CONFIGs to move.
  481. options: option flags.
  482. build_dir: Build directory.
  483. """
  484. self.configs = configs
  485. self.options = options
  486. self.dotconfig = os.path.join(build_dir, '.config')
  487. self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
  488. self.spl_autoconf = os.path.join(build_dir, 'spl', 'include',
  489. 'autoconf.mk')
  490. self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH)
  491. self.defconfig = os.path.join(build_dir, 'defconfig')
  492. def get_arch(self):
  493. """Parse .config file and return the architecture.
  494. Returns:
  495. Architecture name (e.g. 'arm').
  496. """
  497. arch = ''
  498. cpu = ''
  499. for line in open(self.dotconfig):
  500. m = self.re_arch.match(line)
  501. if m:
  502. arch = m.group(1)
  503. continue
  504. m = self.re_cpu.match(line)
  505. if m:
  506. cpu = m.group(1)
  507. if not arch:
  508. return None
  509. # fix-up for aarch64
  510. if arch == 'arm' and cpu == 'armv8':
  511. arch = 'aarch64'
  512. return arch
  513. def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
  514. """Parse .config, defconfig, include/autoconf.mk for one config.
  515. This function looks for the config options in the lines from
  516. defconfig, .config, and include/autoconf.mk in order to decide
  517. which action should be taken for this defconfig.
  518. Arguments:
  519. config: CONFIG name to parse.
  520. dotconfig_lines: lines from the .config file.
  521. autoconf_lines: lines from the include/autoconf.mk file.
  522. Returns:
  523. A tupple of the action for this defconfig and the line
  524. matched for the config.
  525. """
  526. not_set = '# %s is not set' % config
  527. for line in autoconf_lines:
  528. line = line.rstrip()
  529. if line.startswith(config + '='):
  530. new_val = line
  531. break
  532. else:
  533. new_val = not_set
  534. new_val = try_expand(new_val)
  535. for line in dotconfig_lines:
  536. line = line.rstrip()
  537. if line.startswith(config + '=') or line == not_set:
  538. old_val = line
  539. break
  540. else:
  541. if new_val == not_set:
  542. return (ACTION_NO_ENTRY, config)
  543. else:
  544. return (ACTION_NO_ENTRY_WARN, config)
  545. # If this CONFIG is neither bool nor trisate
  546. if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
  547. # tools/scripts/define2mk.sed changes '1' to 'y'.
  548. # This is a problem if the CONFIG is int type.
  549. # Check the type in Kconfig and handle it correctly.
  550. if new_val[-2:] == '=y':
  551. new_val = new_val[:-1] + '1'
  552. return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE,
  553. new_val)
  554. def update_dotconfig(self):
  555. """Parse files for the config options and update the .config.
  556. This function parses the generated .config and include/autoconf.mk
  557. searching the target options.
  558. Move the config option(s) to the .config as needed.
  559. Arguments:
  560. defconfig: defconfig name.
  561. Returns:
  562. Return a tuple of (updated flag, log string).
  563. The "updated flag" is True if the .config was updated, False
  564. otherwise. The "log string" shows what happend to the .config.
  565. """
  566. results = []
  567. updated = False
  568. suspicious = False
  569. rm_files = [self.config_autoconf, self.autoconf]
  570. if self.options.spl:
  571. if os.path.exists(self.spl_autoconf):
  572. autoconf_path = self.spl_autoconf
  573. rm_files.append(self.spl_autoconf)
  574. else:
  575. for f in rm_files:
  576. os.remove(f)
  577. return (updated, suspicious,
  578. color_text(self.options.color, COLOR_BROWN,
  579. "SPL is not enabled. Skipped.") + '\n')
  580. else:
  581. autoconf_path = self.autoconf
  582. with open(self.dotconfig) as f:
  583. dotconfig_lines = f.readlines()
  584. with open(autoconf_path) as f:
  585. autoconf_lines = f.readlines()
  586. for config in self.configs:
  587. result = self.parse_one_config(config, dotconfig_lines,
  588. autoconf_lines)
  589. results.append(result)
  590. log = ''
  591. for (action, value) in results:
  592. if action == ACTION_MOVE:
  593. actlog = "Move '%s'" % value
  594. log_color = COLOR_LIGHT_GREEN
  595. elif action == ACTION_NO_ENTRY:
  596. actlog = "%s is not defined in Kconfig. Do nothing." % value
  597. log_color = COLOR_LIGHT_BLUE
  598. elif action == ACTION_NO_ENTRY_WARN:
  599. actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value
  600. log_color = COLOR_YELLOW
  601. suspicious = True
  602. elif action == ACTION_NO_CHANGE:
  603. actlog = "'%s' is the same as the define in Kconfig. Do nothing." \
  604. % value
  605. log_color = COLOR_LIGHT_PURPLE
  606. elif action == ACTION_SPL_NOT_EXIST:
  607. actlog = "SPL is not enabled for this defconfig. Skip."
  608. log_color = COLOR_PURPLE
  609. else:
  610. sys.exit("Internal Error. This should not happen.")
  611. log += color_text(self.options.color, log_color, actlog) + '\n'
  612. with open(self.dotconfig, 'a') as f:
  613. for (action, value) in results:
  614. if action == ACTION_MOVE:
  615. f.write(value + '\n')
  616. updated = True
  617. self.results = results
  618. for f in rm_files:
  619. os.remove(f)
  620. return (updated, suspicious, log)
  621. def check_defconfig(self):
  622. """Check the defconfig after savedefconfig
  623. Returns:
  624. Return additional log if moved CONFIGs were removed again by
  625. 'make savedefconfig'.
  626. """
  627. log = ''
  628. with open(self.defconfig) as f:
  629. defconfig_lines = f.readlines()
  630. for (action, value) in self.results:
  631. if action != ACTION_MOVE:
  632. continue
  633. if not value + '\n' in defconfig_lines:
  634. log += color_text(self.options.color, COLOR_YELLOW,
  635. "'%s' was removed by savedefconfig.\n" %
  636. value)
  637. return log
  638. class DatabaseThread(threading.Thread):
  639. """This thread processes results from Slot threads.
  640. It collects the data in the master config directary. There is only one
  641. result thread, and this helps to serialise the build output.
  642. """
  643. def __init__(self, config_db, db_queue):
  644. """Set up a new result thread
  645. Args:
  646. builder: Builder which will be sent each result
  647. """
  648. threading.Thread.__init__(self)
  649. self.config_db = config_db
  650. self.db_queue= db_queue
  651. def run(self):
  652. """Called to start up the result thread.
  653. We collect the next result job and pass it on to the build.
  654. """
  655. while True:
  656. defconfig, configs = self.db_queue.get()
  657. self.config_db[defconfig] = configs
  658. self.db_queue.task_done()
  659. class Slot:
  660. """A slot to store a subprocess.
  661. Each instance of this class handles one subprocess.
  662. This class is useful to control multiple threads
  663. for faster processing.
  664. """
  665. def __init__(self, toolchains, configs, options, progress, devnull,
  666. make_cmd, reference_src_dir, db_queue):
  667. """Create a new process slot.
  668. Arguments:
  669. toolchains: Toolchains object containing toolchains.
  670. configs: A list of CONFIGs to move.
  671. options: option flags.
  672. progress: A progress indicator.
  673. devnull: A file object of '/dev/null'.
  674. make_cmd: command name of GNU Make.
  675. reference_src_dir: Determine the true starting config state from this
  676. source tree.
  677. db_queue: output queue to write config info for the database
  678. """
  679. self.toolchains = toolchains
  680. self.options = options
  681. self.progress = progress
  682. self.build_dir = tempfile.mkdtemp()
  683. self.devnull = devnull
  684. self.make_cmd = (make_cmd, 'O=' + self.build_dir)
  685. self.reference_src_dir = reference_src_dir
  686. self.db_queue = db_queue
  687. self.parser = KconfigParser(configs, options, self.build_dir)
  688. self.state = STATE_IDLE
  689. self.failed_boards = set()
  690. self.suspicious_boards = set()
  691. def __del__(self):
  692. """Delete the working directory
  693. This function makes sure the temporary directory is cleaned away
  694. even if Python suddenly dies due to error. It should be done in here
  695. because it is guaranteed the destructor is always invoked when the
  696. instance of the class gets unreferenced.
  697. If the subprocess is still running, wait until it finishes.
  698. """
  699. if self.state != STATE_IDLE:
  700. while self.ps.poll() == None:
  701. pass
  702. shutil.rmtree(self.build_dir)
  703. def add(self, defconfig):
  704. """Assign a new subprocess for defconfig and add it to the slot.
  705. If the slot is vacant, create a new subprocess for processing the
  706. given defconfig and add it to the slot. Just returns False if
  707. the slot is occupied (i.e. the current subprocess is still running).
  708. Arguments:
  709. defconfig: defconfig name.
  710. Returns:
  711. Return True on success or False on failure
  712. """
  713. if self.state != STATE_IDLE:
  714. return False
  715. self.defconfig = defconfig
  716. self.log = ''
  717. self.current_src_dir = self.reference_src_dir
  718. self.do_defconfig()
  719. return True
  720. def poll(self):
  721. """Check the status of the subprocess and handle it as needed.
  722. Returns True if the slot is vacant (i.e. in idle state).
  723. If the configuration is successfully finished, assign a new
  724. subprocess to build include/autoconf.mk.
  725. If include/autoconf.mk is generated, invoke the parser to
  726. parse the .config and the include/autoconf.mk, moving
  727. config options to the .config as needed.
  728. If the .config was updated, run "make savedefconfig" to sync
  729. it, update the original defconfig, and then set the slot back
  730. to the idle state.
  731. Returns:
  732. Return True if the subprocess is terminated, False otherwise
  733. """
  734. if self.state == STATE_IDLE:
  735. return True
  736. if self.ps.poll() == None:
  737. return False
  738. if self.ps.poll() != 0:
  739. self.handle_error()
  740. elif self.state == STATE_DEFCONFIG:
  741. if self.reference_src_dir and not self.current_src_dir:
  742. self.do_savedefconfig()
  743. else:
  744. self.do_autoconf()
  745. elif self.state == STATE_AUTOCONF:
  746. if self.current_src_dir:
  747. self.current_src_dir = None
  748. self.do_defconfig()
  749. elif self.options.build_db:
  750. self.do_build_db()
  751. else:
  752. self.do_savedefconfig()
  753. elif self.state == STATE_SAVEDEFCONFIG:
  754. self.update_defconfig()
  755. else:
  756. sys.exit("Internal Error. This should not happen.")
  757. return True if self.state == STATE_IDLE else False
  758. def handle_error(self):
  759. """Handle error cases."""
  760. self.log += color_text(self.options.color, COLOR_LIGHT_RED,
  761. "Failed to process.\n")
  762. if self.options.verbose:
  763. self.log += color_text(self.options.color, COLOR_LIGHT_CYAN,
  764. self.ps.stderr.read().decode())
  765. self.finish(False)
  766. def do_defconfig(self):
  767. """Run 'make <board>_defconfig' to create the .config file."""
  768. cmd = list(self.make_cmd)
  769. cmd.append(self.defconfig)
  770. self.ps = subprocess.Popen(cmd, stdout=self.devnull,
  771. stderr=subprocess.PIPE,
  772. cwd=self.current_src_dir)
  773. self.state = STATE_DEFCONFIG
  774. def do_autoconf(self):
  775. """Run 'make AUTO_CONF_PATH'."""
  776. arch = self.parser.get_arch()
  777. try:
  778. toolchain = self.toolchains.Select(arch)
  779. except ValueError:
  780. self.log += color_text(self.options.color, COLOR_YELLOW,
  781. "Tool chain for '%s' is missing. Do nothing.\n" % arch)
  782. self.finish(False)
  783. return
  784. env = toolchain.MakeEnvironment(False)
  785. cmd = list(self.make_cmd)
  786. cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
  787. cmd.append(AUTO_CONF_PATH)
  788. self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env,
  789. stderr=subprocess.PIPE,
  790. cwd=self.current_src_dir)
  791. self.state = STATE_AUTOCONF
  792. def do_build_db(self):
  793. """Add the board to the database"""
  794. configs = {}
  795. with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd:
  796. for line in fd.readlines():
  797. if line.startswith('CONFIG'):
  798. config, value = line.split('=', 1)
  799. configs[config] = value.rstrip()
  800. self.db_queue.put([self.defconfig, configs])
  801. self.finish(True)
  802. def do_savedefconfig(self):
  803. """Update the .config and run 'make savedefconfig'."""
  804. (updated, suspicious, log) = self.parser.update_dotconfig()
  805. if suspicious:
  806. self.suspicious_boards.add(self.defconfig)
  807. self.log += log
  808. if not self.options.force_sync and not updated:
  809. self.finish(True)
  810. return
  811. if updated:
  812. self.log += color_text(self.options.color, COLOR_LIGHT_GREEN,
  813. "Syncing by savedefconfig...\n")
  814. else:
  815. self.log += "Syncing by savedefconfig (forced by option)...\n"
  816. cmd = list(self.make_cmd)
  817. cmd.append('savedefconfig')
  818. self.ps = subprocess.Popen(cmd, stdout=self.devnull,
  819. stderr=subprocess.PIPE)
  820. self.state = STATE_SAVEDEFCONFIG
  821. def update_defconfig(self):
  822. """Update the input defconfig and go back to the idle state."""
  823. log = self.parser.check_defconfig()
  824. if log:
  825. self.suspicious_boards.add(self.defconfig)
  826. self.log += log
  827. orig_defconfig = os.path.join('configs', self.defconfig)
  828. new_defconfig = os.path.join(self.build_dir, 'defconfig')
  829. updated = not filecmp.cmp(orig_defconfig, new_defconfig)
  830. if updated:
  831. self.log += color_text(self.options.color, COLOR_LIGHT_BLUE,
  832. "defconfig was updated.\n")
  833. if not self.options.dry_run and updated:
  834. shutil.move(new_defconfig, orig_defconfig)
  835. self.finish(True)
  836. def finish(self, success):
  837. """Display log along with progress and go to the idle state.
  838. Arguments:
  839. success: Should be True when the defconfig was processed
  840. successfully, or False when it fails.
  841. """
  842. # output at least 30 characters to hide the "* defconfigs out of *".
  843. log = self.defconfig.ljust(30) + '\n'
  844. log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
  845. # Some threads are running in parallel.
  846. # Print log atomically to not mix up logs from different threads.
  847. print(log, file=(sys.stdout if success else sys.stderr))
  848. if not success:
  849. if self.options.exit_on_error:
  850. sys.exit("Exit on error.")
  851. # If --exit-on-error flag is not set, skip this board and continue.
  852. # Record the failed board.
  853. self.failed_boards.add(self.defconfig)
  854. self.progress.inc()
  855. self.progress.show()
  856. self.state = STATE_IDLE
  857. def get_failed_boards(self):
  858. """Returns a set of failed boards (defconfigs) in this slot.
  859. """
  860. return self.failed_boards
  861. def get_suspicious_boards(self):
  862. """Returns a set of boards (defconfigs) with possible misconversion.
  863. """
  864. return self.suspicious_boards - self.failed_boards
  865. class Slots:
  866. """Controller of the array of subprocess slots."""
  867. def __init__(self, toolchains, configs, options, progress,
  868. reference_src_dir, db_queue):
  869. """Create a new slots controller.
  870. Arguments:
  871. toolchains: Toolchains object containing toolchains.
  872. configs: A list of CONFIGs to move.
  873. options: option flags.
  874. progress: A progress indicator.
  875. reference_src_dir: Determine the true starting config state from this
  876. source tree.
  877. db_queue: output queue to write config info for the database
  878. """
  879. self.options = options
  880. self.slots = []
  881. devnull = get_devnull()
  882. make_cmd = get_make_cmd()
  883. for i in range(options.jobs):
  884. self.slots.append(Slot(toolchains, configs, options, progress,
  885. devnull, make_cmd, reference_src_dir,
  886. db_queue))
  887. def add(self, defconfig):
  888. """Add a new subprocess if a vacant slot is found.
  889. Arguments:
  890. defconfig: defconfig name to be put into.
  891. Returns:
  892. Return True on success or False on failure
  893. """
  894. for slot in self.slots:
  895. if slot.add(defconfig):
  896. return True
  897. return False
  898. def available(self):
  899. """Check if there is a vacant slot.
  900. Returns:
  901. Return True if at lease one vacant slot is found, False otherwise.
  902. """
  903. for slot in self.slots:
  904. if slot.poll():
  905. return True
  906. return False
  907. def empty(self):
  908. """Check if all slots are vacant.
  909. Returns:
  910. Return True if all the slots are vacant, False otherwise.
  911. """
  912. ret = True
  913. for slot in self.slots:
  914. if not slot.poll():
  915. ret = False
  916. return ret
  917. def show_failed_boards(self):
  918. """Display all of the failed boards (defconfigs)."""
  919. boards = set()
  920. output_file = 'moveconfig.failed'
  921. for slot in self.slots:
  922. boards |= slot.get_failed_boards()
  923. if boards:
  924. boards = '\n'.join(boards) + '\n'
  925. msg = "The following boards were not processed due to error:\n"
  926. msg += boards
  927. msg += "(the list has been saved in %s)\n" % output_file
  928. print(color_text(self.options.color, COLOR_LIGHT_RED,
  929. msg), file=sys.stderr)
  930. with open(output_file, 'w') as f:
  931. f.write(boards)
  932. def show_suspicious_boards(self):
  933. """Display all boards (defconfigs) with possible misconversion."""
  934. boards = set()
  935. output_file = 'moveconfig.suspicious'
  936. for slot in self.slots:
  937. boards |= slot.get_suspicious_boards()
  938. if boards:
  939. boards = '\n'.join(boards) + '\n'
  940. msg = "The following boards might have been converted incorrectly.\n"
  941. msg += "It is highly recommended to check them manually:\n"
  942. msg += boards
  943. msg += "(the list has been saved in %s)\n" % output_file
  944. print(color_text(self.options.color, COLOR_YELLOW,
  945. msg), file=sys.stderr)
  946. with open(output_file, 'w') as f:
  947. f.write(boards)
  948. class ReferenceSource:
  949. """Reference source against which original configs should be parsed."""
  950. def __init__(self, commit):
  951. """Create a reference source directory based on a specified commit.
  952. Arguments:
  953. commit: commit to git-clone
  954. """
  955. self.src_dir = tempfile.mkdtemp()
  956. print("Cloning git repo to a separate work directory...")
  957. subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
  958. cwd=self.src_dir)
  959. print("Checkout '%s' to build the original autoconf.mk." % \
  960. subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip())
  961. subprocess.check_output(['git', 'checkout', commit],
  962. stderr=subprocess.STDOUT, cwd=self.src_dir)
  963. def __del__(self):
  964. """Delete the reference source directory
  965. This function makes sure the temporary directory is cleaned away
  966. even if Python suddenly dies due to error. It should be done in here
  967. because it is guaranteed the destructor is always invoked when the
  968. instance of the class gets unreferenced.
  969. """
  970. shutil.rmtree(self.src_dir)
  971. def get_dir(self):
  972. """Return the absolute path to the reference source directory."""
  973. return self.src_dir
  974. def move_config(toolchains, configs, options, db_queue):
  975. """Move config options to defconfig files.
  976. Arguments:
  977. configs: A list of CONFIGs to move.
  978. options: option flags
  979. """
  980. if len(configs) == 0:
  981. if options.force_sync:
  982. print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ')
  983. elif options.build_db:
  984. print('Building %s database' % CONFIG_DATABASE)
  985. else:
  986. print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ')
  987. else:
  988. print('Move ' + ', '.join(configs), end=' ')
  989. print('(jobs: %d)\n' % options.jobs)
  990. if options.git_ref:
  991. reference_src = ReferenceSource(options.git_ref)
  992. reference_src_dir = reference_src.get_dir()
  993. else:
  994. reference_src_dir = None
  995. if options.defconfigs:
  996. defconfigs = get_matched_defconfigs(options.defconfigs)
  997. else:
  998. defconfigs = get_all_defconfigs()
  999. progress = Progress(len(defconfigs))
  1000. slots = Slots(toolchains, configs, options, progress, reference_src_dir,
  1001. db_queue)
  1002. # Main loop to process defconfig files:
  1003. # Add a new subprocess into a vacant slot.
  1004. # Sleep if there is no available slot.
  1005. for defconfig in defconfigs:
  1006. while not slots.add(defconfig):
  1007. while not slots.available():
  1008. # No available slot: sleep for a while
  1009. time.sleep(SLEEP_TIME)
  1010. # wait until all the subprocesses finish
  1011. while not slots.empty():
  1012. time.sleep(SLEEP_TIME)
  1013. print('')
  1014. slots.show_failed_boards()
  1015. slots.show_suspicious_boards()
  1016. def find_kconfig_rules(kconf, config, imply_config):
  1017. """Check whether a config has a 'select' or 'imply' keyword
  1018. Args:
  1019. kconf: Kconfiglib.Kconfig object
  1020. config: Name of config to check (without CONFIG_ prefix)
  1021. imply_config: Implying config (without CONFIG_ prefix) which may or
  1022. may not have an 'imply' for 'config')
  1023. Returns:
  1024. Symbol object for 'config' if found, else None
  1025. """
  1026. sym = kconf.syms.get(imply_config)
  1027. if sym:
  1028. for sel, cond in (sym.selects + sym.implies):
  1029. if sel == config:
  1030. return sym
  1031. return None
  1032. def check_imply_rule(kconf, config, imply_config):
  1033. """Check if we can add an 'imply' option
  1034. This finds imply_config in the Kconfig and looks to see if it is possible
  1035. to add an 'imply' for 'config' to that part of the Kconfig.
  1036. Args:
  1037. kconf: Kconfiglib.Kconfig object
  1038. config: Name of config to check (without CONFIG_ prefix)
  1039. imply_config: Implying config (without CONFIG_ prefix) which may or
  1040. may not have an 'imply' for 'config')
  1041. Returns:
  1042. tuple:
  1043. filename of Kconfig file containing imply_config, or None if none
  1044. line number within the Kconfig file, or 0 if none
  1045. message indicating the result
  1046. """
  1047. sym = kconf.syms.get(imply_config)
  1048. if not sym:
  1049. return 'cannot find sym'
  1050. nodes = sym.nodes
  1051. if len(nodes) != 1:
  1052. return '%d locations' % len(nodes)
  1053. fname, linenum = nodes[0].filename, nodes[0].linern
  1054. cwd = os.getcwd()
  1055. if cwd and fname.startswith(cwd):
  1056. fname = fname[len(cwd) + 1:]
  1057. file_line = ' at %s:%d' % (fname, linenum)
  1058. with open(fname) as fd:
  1059. data = fd.read().splitlines()
  1060. if data[linenum - 1] != 'config %s' % imply_config:
  1061. return None, 0, 'bad sym format %s%s' % (data[linenum], file_line)
  1062. return fname, linenum, 'adding%s' % file_line
  1063. def add_imply_rule(config, fname, linenum):
  1064. """Add a new 'imply' option to a Kconfig
  1065. Args:
  1066. config: config option to add an imply for (without CONFIG_ prefix)
  1067. fname: Kconfig filename to update
  1068. linenum: Line number to place the 'imply' before
  1069. Returns:
  1070. Message indicating the result
  1071. """
  1072. file_line = ' at %s:%d' % (fname, linenum)
  1073. data = open(fname).read().splitlines()
  1074. linenum -= 1
  1075. for offset, line in enumerate(data[linenum:]):
  1076. if line.strip().startswith('help') or not line:
  1077. data.insert(linenum + offset, '\timply %s' % config)
  1078. with open(fname, 'w') as fd:
  1079. fd.write('\n'.join(data) + '\n')
  1080. return 'added%s' % file_line
  1081. return 'could not insert%s'
  1082. (IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = (
  1083. 1, 2, 4, 8)
  1084. IMPLY_FLAGS = {
  1085. 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'],
  1086. 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'],
  1087. 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'],
  1088. 'non-arch-board': [
  1089. IMPLY_NON_ARCH_BOARD,
  1090. 'Allow Kconfig options outside arch/ and /board/ to imply'],
  1091. };
  1092. def do_imply_config(config_list, add_imply, imply_flags, skip_added,
  1093. check_kconfig=True, find_superset=False):
  1094. """Find CONFIG options which imply those in the list
  1095. Some CONFIG options can be implied by others and this can help to reduce
  1096. the size of the defconfig files. For example, CONFIG_X86 implies
  1097. CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and
  1098. all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to
  1099. each of the x86 defconfig files.
  1100. This function uses the moveconfig database to find such options. It
  1101. displays a list of things that could possibly imply those in the list.
  1102. The algorithm ignores any that start with CONFIG_TARGET since these
  1103. typically refer to only a few defconfigs (often one). It also does not
  1104. display a config with less than 5 defconfigs.
  1105. The algorithm works using sets. For each target config in config_list:
  1106. - Get the set 'defconfigs' which use that target config
  1107. - For each config (from a list of all configs):
  1108. - Get the set 'imply_defconfig' of defconfigs which use that config
  1109. -
  1110. - If imply_defconfigs contains anything not in defconfigs then
  1111. this config does not imply the target config
  1112. Params:
  1113. config_list: List of CONFIG options to check (each a string)
  1114. add_imply: Automatically add an 'imply' for each config.
  1115. imply_flags: Flags which control which implying configs are allowed
  1116. (IMPLY_...)
  1117. skip_added: Don't show options which already have an imply added.
  1118. check_kconfig: Check if implied symbols already have an 'imply' or
  1119. 'select' for the target config, and show this information if so.
  1120. find_superset: True to look for configs which are a superset of those
  1121. already found. So for example if CONFIG_EXYNOS5 implies an option,
  1122. but CONFIG_EXYNOS covers a larger set of defconfigs and also
  1123. implies that option, this will drop the former in favour of the
  1124. latter. In practice this option has not proved very used.
  1125. Note the terminoloy:
  1126. config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM')
  1127. defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig')
  1128. """
  1129. kconf = KconfigScanner().conf if check_kconfig else None
  1130. if add_imply and add_imply != 'all':
  1131. add_imply = add_imply.split()
  1132. # key is defconfig name, value is dict of (CONFIG_xxx, value)
  1133. config_db = {}
  1134. # Holds a dict containing the set of defconfigs that contain each config
  1135. # key is config, value is set of defconfigs using that config
  1136. defconfig_db = collections.defaultdict(set)
  1137. # Set of all config options we have seen
  1138. all_configs = set()
  1139. # Set of all defconfigs we have seen
  1140. all_defconfigs = set()
  1141. # Read in the database
  1142. configs = {}
  1143. with open(CONFIG_DATABASE) as fd:
  1144. for line in fd.readlines():
  1145. line = line.rstrip()
  1146. if not line: # Separator between defconfigs
  1147. config_db[defconfig] = configs
  1148. all_defconfigs.add(defconfig)
  1149. configs = {}
  1150. elif line[0] == ' ': # CONFIG line
  1151. config, value = line.strip().split('=', 1)
  1152. configs[config] = value
  1153. defconfig_db[config].add(defconfig)
  1154. all_configs.add(config)
  1155. else: # New defconfig
  1156. defconfig = line
  1157. # Work through each target config option in tern, independently
  1158. for config in config_list:
  1159. defconfigs = defconfig_db.get(config)
  1160. if not defconfigs:
  1161. print('%s not found in any defconfig' % config)
  1162. continue
  1163. # Get the set of defconfigs without this one (since a config cannot
  1164. # imply itself)
  1165. non_defconfigs = all_defconfigs - defconfigs
  1166. num_defconfigs = len(defconfigs)
  1167. print('%s found in %d/%d defconfigs' % (config, num_defconfigs,
  1168. len(all_configs)))
  1169. # This will hold the results: key=config, value=defconfigs containing it
  1170. imply_configs = {}
  1171. rest_configs = all_configs - set([config])
  1172. # Look at every possible config, except the target one
  1173. for imply_config in rest_configs:
  1174. if 'ERRATUM' in imply_config:
  1175. continue
  1176. if not (imply_flags & IMPLY_CMD):
  1177. if 'CONFIG_CMD' in imply_config:
  1178. continue
  1179. if not (imply_flags & IMPLY_TARGET):
  1180. if 'CONFIG_TARGET' in imply_config:
  1181. continue
  1182. # Find set of defconfigs that have this config
  1183. imply_defconfig = defconfig_db[imply_config]
  1184. # Get the intersection of this with defconfigs containing the
  1185. # target config
  1186. common_defconfigs = imply_defconfig & defconfigs
  1187. # Get the set of defconfigs containing this config which DO NOT
  1188. # also contain the taret config. If this set is non-empty it means
  1189. # that this config affects other defconfigs as well as (possibly)
  1190. # the ones affected by the target config. This means it implies
  1191. # things we don't want to imply.
  1192. not_common_defconfigs = imply_defconfig & non_defconfigs
  1193. if not_common_defconfigs:
  1194. continue
  1195. # If there are common defconfigs, imply_config may be useful
  1196. if common_defconfigs:
  1197. skip = False
  1198. if find_superset:
  1199. for prev in list(imply_configs.keys()):
  1200. prev_count = len(imply_configs[prev])
  1201. count = len(common_defconfigs)
  1202. if (prev_count > count and
  1203. (imply_configs[prev] & common_defconfigs ==
  1204. common_defconfigs)):
  1205. # skip imply_config because prev is a superset
  1206. skip = True
  1207. break
  1208. elif count > prev_count:
  1209. # delete prev because imply_config is a superset
  1210. del imply_configs[prev]
  1211. if not skip:
  1212. imply_configs[imply_config] = common_defconfigs
  1213. # Now we have a dict imply_configs of configs which imply each config
  1214. # The value of each dict item is the set of defconfigs containing that
  1215. # config. Rank them so that we print the configs that imply the largest
  1216. # number of defconfigs first.
  1217. ranked_iconfigs = sorted(imply_configs,
  1218. key=lambda k: len(imply_configs[k]), reverse=True)
  1219. kconfig_info = ''
  1220. cwd = os.getcwd()
  1221. add_list = collections.defaultdict(list)
  1222. for iconfig in ranked_iconfigs:
  1223. num_common = len(imply_configs[iconfig])
  1224. # Don't bother if there are less than 5 defconfigs affected.
  1225. if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5):
  1226. continue
  1227. missing = defconfigs - imply_configs[iconfig]
  1228. missing_str = ', '.join(missing) if missing else 'all'
  1229. missing_str = ''
  1230. show = True
  1231. if kconf:
  1232. sym = find_kconfig_rules(kconf, config[CONFIG_LEN:],
  1233. iconfig[CONFIG_LEN:])
  1234. kconfig_info = ''
  1235. if sym:
  1236. nodes = sym.nodes
  1237. if len(nodes) == 1:
  1238. fname, linenum = nodes[0].filename, nodes[0].linenr
  1239. if cwd and fname.startswith(cwd):
  1240. fname = fname[len(cwd) + 1:]
  1241. kconfig_info = '%s:%d' % (fname, linenum)
  1242. if skip_added:
  1243. show = False
  1244. else:
  1245. sym = kconf.syms.get(iconfig[CONFIG_LEN:])
  1246. fname = ''
  1247. if sym:
  1248. nodes = sym.nodes
  1249. if len(nodes) == 1:
  1250. fname, linenum = nodes[0].filename, nodes[0].linenr
  1251. if cwd and fname.startswith(cwd):
  1252. fname = fname[len(cwd) + 1:]
  1253. in_arch_board = not sym or (fname.startswith('arch') or
  1254. fname.startswith('board'))
  1255. if (not in_arch_board and
  1256. not (imply_flags & IMPLY_NON_ARCH_BOARD)):
  1257. continue
  1258. if add_imply and (add_imply == 'all' or
  1259. iconfig in add_imply):
  1260. fname, linenum, kconfig_info = (check_imply_rule(kconf,
  1261. config[CONFIG_LEN:], iconfig[CONFIG_LEN:]))
  1262. if fname:
  1263. add_list[fname].append(linenum)
  1264. if show and kconfig_info != 'skip':
  1265. print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30),
  1266. kconfig_info, missing_str))
  1267. # Having collected a list of things to add, now we add them. We process
  1268. # each file from the largest line number to the smallest so that
  1269. # earlier additions do not affect our line numbers. E.g. if we added an
  1270. # imply at line 20 it would change the position of each line after
  1271. # that.
  1272. for fname, linenums in add_list.items():
  1273. for linenum in sorted(linenums, reverse=True):
  1274. add_imply_rule(config[CONFIG_LEN:], fname, linenum)
  1275. def main():
  1276. try:
  1277. cpu_count = multiprocessing.cpu_count()
  1278. except NotImplementedError:
  1279. cpu_count = 1
  1280. parser = optparse.OptionParser()
  1281. # Add options here
  1282. parser.add_option('-a', '--add-imply', type='string', default='',
  1283. help='comma-separated list of CONFIG options to add '
  1284. "an 'imply' statement to for the CONFIG in -i")
  1285. parser.add_option('-A', '--skip-added', action='store_true', default=False,
  1286. help="don't show options which are already marked as "
  1287. 'implying others')
  1288. parser.add_option('-b', '--build-db', action='store_true', default=False,
  1289. help='build a CONFIG database')
  1290. parser.add_option('-c', '--color', action='store_true', default=False,
  1291. help='display the log in color')
  1292. parser.add_option('-C', '--commit', action='store_true', default=False,
  1293. help='Create a git commit for the operation')
  1294. parser.add_option('-d', '--defconfigs', type='string',
  1295. help='a file containing a list of defconfigs to move, '
  1296. "one per line (for example 'snow_defconfig') "
  1297. "or '-' to read from stdin")
  1298. parser.add_option('-i', '--imply', action='store_true', default=False,
  1299. help='find options which imply others')
  1300. parser.add_option('-I', '--imply-flags', type='string', default='',
  1301. help="control the -i option ('help' for help")
  1302. parser.add_option('-n', '--dry-run', action='store_true', default=False,
  1303. help='perform a trial run (show log with no changes)')
  1304. parser.add_option('-e', '--exit-on-error', action='store_true',
  1305. default=False,
  1306. help='exit immediately on any error')
  1307. parser.add_option('-s', '--force-sync', action='store_true', default=False,
  1308. help='force sync by savedefconfig')
  1309. parser.add_option('-S', '--spl', action='store_true', default=False,
  1310. help='parse config options defined for SPL build')
  1311. parser.add_option('-H', '--headers-only', dest='cleanup_headers_only',
  1312. action='store_true', default=False,
  1313. help='only cleanup the headers')
  1314. parser.add_option('-j', '--jobs', type='int', default=cpu_count,
  1315. help='the number of jobs to run simultaneously')
  1316. parser.add_option('-r', '--git-ref', type='string',
  1317. help='the git ref to clone for building the autoconf.mk')
  1318. parser.add_option('-y', '--yes', action='store_true', default=False,
  1319. help="respond 'yes' to any prompts")
  1320. parser.add_option('-v', '--verbose', action='store_true', default=False,
  1321. help='show any build errors as boards are built')
  1322. parser.usage += ' CONFIG ...'
  1323. (options, configs) = parser.parse_args()
  1324. if len(configs) == 0 and not any((options.force_sync, options.build_db,
  1325. options.imply)):
  1326. parser.print_usage()
  1327. sys.exit(1)
  1328. # prefix the option name with CONFIG_ if missing
  1329. configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config
  1330. for config in configs ]
  1331. check_top_directory()
  1332. if options.imply:
  1333. imply_flags = 0
  1334. if options.imply_flags == 'all':
  1335. imply_flags = -1
  1336. elif options.imply_flags:
  1337. for flag in options.imply_flags.split(','):
  1338. bad = flag not in IMPLY_FLAGS
  1339. if bad:
  1340. print("Invalid flag '%s'" % flag)
  1341. if flag == 'help' or bad:
  1342. print("Imply flags: (separate with ',')")
  1343. for name, info in IMPLY_FLAGS.items():
  1344. print(' %-15s: %s' % (name, info[1]))
  1345. parser.print_usage()
  1346. sys.exit(1)
  1347. imply_flags |= IMPLY_FLAGS[flag][0]
  1348. do_imply_config(configs, options.add_imply, imply_flags,
  1349. options.skip_added)
  1350. return
  1351. config_db = {}
  1352. db_queue = queue.Queue()
  1353. t = DatabaseThread(config_db, db_queue)
  1354. t.setDaemon(True)
  1355. t.start()
  1356. if not options.cleanup_headers_only:
  1357. check_clean_directory()
  1358. bsettings.Setup('')
  1359. toolchains = toolchain.Toolchains()
  1360. toolchains.GetSettings()
  1361. toolchains.Scan(verbose=False)
  1362. move_config(toolchains, configs, options, db_queue)
  1363. db_queue.join()
  1364. if configs:
  1365. cleanup_headers(configs, options)
  1366. cleanup_extra_options(configs, options)
  1367. cleanup_whitelist(configs, options)
  1368. cleanup_readme(configs, options)
  1369. if options.commit:
  1370. subprocess.call(['git', 'add', '-u'])
  1371. if configs:
  1372. msg = 'Convert %s %sto Kconfig' % (configs[0],
  1373. 'et al ' if len(configs) > 1 else '')
  1374. msg += ('\n\nThis converts the following to Kconfig:\n %s\n' %
  1375. '\n '.join(configs))
  1376. else:
  1377. msg = 'configs: Resync with savedefconfig'
  1378. msg += '\n\nRsync all defconfig files using moveconfig.py'
  1379. subprocess.call(['git', 'commit', '-s', '-m', msg])
  1380. if options.build_db:
  1381. with open(CONFIG_DATABASE, 'w') as fd:
  1382. for defconfig, configs in config_db.items():
  1383. fd.write('%s\n' % defconfig)
  1384. for config in sorted(configs.keys()):
  1385. fd.write(' %s=%s\n' % (config, configs[config]))
  1386. fd.write('\n')
  1387. if __name__ == '__main__':
  1388. main()