convert-overrides.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. #!/usr/bin/env python3
  2. #
  3. # Conversion script to add new override syntax to existing bitbake metadata
  4. #
  5. # Copyright (C) 2021 Richard Purdie
  6. #
  7. # SPDX-License-Identifier: GPL-2.0-only
  8. #
  9. #
  10. # To use this script on a new layer you need to list the overrides the
  11. # layer is known to use in the list below.
  12. #
  13. # Known constraint: Matching is 'loose' and in particular will find variable
  14. # and function names with "_append" and "_remove" in them. Those need to be
  15. # filtered out manually or in the skip list below.
  16. #
  17. import re
  18. import os
  19. import sys
  20. import tempfile
  21. import shutil
  22. import mimetypes
  23. if len(sys.argv) < 2:
  24. print("Please specify a directory to run the conversion script against.")
  25. sys.exit(1)
  26. # List of strings to treat as overrides
  27. vars = ["append", "prepend", "remove"]
  28. vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
  29. vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"]
  30. vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
  31. vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
  32. vars = vars + ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
  33. vars = vars + ["tune-", "pn-", "forcevariable"]
  34. vars = vars + ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
  35. vars = vars + ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
  36. vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
  37. vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
  38. vars = vars + ["linux-gnueabi", "eabi"]
  39. vars = vars + ["virtclass-multilib", "virtclass-mcextend"]
  40. # List of strings to treat as overrides but only with whitespace following or another override (more restricted matching).
  41. # Handles issues with arc matching arch.
  42. shortvars = ["arc", "mips", "mipsel", "sh4"]
  43. # Variables which take packagenames as an override
  44. packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY", "DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY",
  45. "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst", "INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE",
  46. "PKGE", "PKGV", "PKGR", "USERADD_PARAM", "GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst",
  47. "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM", "SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA",
  48. "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"]
  49. # Expressions to skip if encountered, these are not overrides
  50. skips = ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
  51. skips = skips + ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
  52. skips = skips + ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
  53. skips = skips + ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
  54. skips = skips + ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
  55. skips = skips + ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
  56. skips = skips + ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
  57. skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
  58. skips = skips + ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
  59. imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"]
  60. packagevars = packagevars + imagevars
  61. vars_re = {}
  62. for exp in vars:
  63. vars_re[exp] = (re.compile('((^|[#\'"\s\-\+])[A-Za-z0-9_\-:${}\.]+)_' + exp), r"\1:" + exp)
  64. shortvars_re = {}
  65. for exp in shortvars:
  66. shortvars_re[exp] = (re.compile('((^|[#\'"\s\-\+])[A-Za-z0-9_\-:${}\.]+)_' + exp + '([\(\'"\s:])'), r"\1:" + exp + r"\3")
  67. package_re = {}
  68. for exp in packagevars:
  69. package_re[exp] = (re.compile('(^|[#\'"\s\-\+]+)' + exp + '_' + '([$a-z"\'\s%\[<{\\\*].)'), r"\1" + exp + r":\2")
  70. # Other substitutions to make
  71. subs = {
  72. 'r = re.compile("([^:]+):\s*(.*)")' : 'r = re.compile("(^.+?):\s+(.*)")',
  73. "val = d.getVar('%s_%s' % (var, pkg))" : "val = d.getVar('%s:%s' % (var, pkg))",
  74. "f.write('%s_%s: %s\\n' % (var, pkg, encode(val)))" : "f.write('%s:%s: %s\\n' % (var, pkg, encode(val)))",
  75. "d.getVar('%s_%s' % (scriptlet_name, pkg))" : "d.getVar('%s:%s' % (scriptlet_name, pkg))",
  76. 'ret.append(v + "_" + p)' : 'ret.append(v + ":" + p)',
  77. }
  78. def processfile(fn):
  79. print("processing file '%s'" % fn)
  80. try:
  81. fh, abs_path = tempfile.mkstemp()
  82. with os.fdopen(fh, 'w') as new_file:
  83. with open(fn, "r") as old_file:
  84. for line in old_file:
  85. skip = False
  86. for s in skips:
  87. if s in line:
  88. skip = True
  89. if "ptest_append" in line or "ptest_remove" in line or "ptest_prepend" in line:
  90. skip = False
  91. for sub in subs:
  92. if sub in line:
  93. line = line.replace(sub, subs[sub])
  94. skip = True
  95. if not skip:
  96. for pvar in packagevars:
  97. line = package_re[pvar][0].sub(package_re[pvar][1], line)
  98. for var in vars:
  99. line = vars_re[var][0].sub(vars_re[var][1], line)
  100. for shortvar in shortvars:
  101. line = shortvars_re[shortvar][0].sub(shortvars_re[shortvar][1], line)
  102. if "pkg_postinst:ontarget" in line:
  103. line = line.replace("pkg_postinst:ontarget", "pkg_postinst_ontarget")
  104. new_file.write(line)
  105. shutil.copymode(fn, abs_path)
  106. os.remove(fn)
  107. shutil.move(abs_path, fn)
  108. except UnicodeDecodeError:
  109. pass
  110. ourname = os.path.basename(sys.argv[0])
  111. ourversion = "0.9.3"
  112. if os.path.isfile(sys.argv[1]):
  113. processfile(sys.argv[1])
  114. sys.exit(0)
  115. for targetdir in sys.argv[1:]:
  116. print("processing directory '%s'" % targetdir)
  117. for root, dirs, files in os.walk(targetdir):
  118. for name in files:
  119. if name == ourname:
  120. continue
  121. fn = os.path.join(root, name)
  122. if os.path.islink(fn):
  123. continue
  124. if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"):
  125. continue
  126. processfile(fn)
  127. print("All files processed with version %s" % ourversion)