create_npm.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255
  1. # Copyright (C) 2016 Intel Corporation
  2. # Copyright (C) 2020 Savoir-Faire Linux
  3. #
  4. # SPDX-License-Identifier: GPL-2.0-only
  5. #
  6. """Recipe creation tool - npm module support plugin"""
  7. import json
  8. import os
  9. import re
  10. import sys
  11. import tempfile
  12. import bb
  13. from bb.fetch2.npm import NpmEnvironment
  14. from bb.fetch2.npmsw import foreach_dependencies
  15. from recipetool.create import RecipeHandler
  16. from recipetool.create import guess_license
  17. from recipetool.create import split_pkg_licenses
  18. TINFOIL = None
  19. def tinfoil_init(instance):
  20. """Initialize tinfoil"""
  21. global TINFOIL
  22. TINFOIL = instance
  23. class NpmRecipeHandler(RecipeHandler):
  24. """Class to handle the npm recipe creation"""
  25. @staticmethod
  26. def _npm_name(name):
  27. """Generate a Yocto friendly npm name"""
  28. name = re.sub("/", "-", name)
  29. name = name.lower()
  30. name = re.sub(r"[^\-a-z0-9]", "", name)
  31. name = name.strip("-")
  32. return name
  33. @staticmethod
  34. def _get_registry(lines):
  35. """Get the registry value from the 'npm://registry' url"""
  36. registry = None
  37. def _handle_registry(varname, origvalue, op, newlines):
  38. nonlocal registry
  39. if origvalue.startswith("npm://"):
  40. registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
  41. return origvalue, None, 0, True
  42. bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)
  43. return registry
  44. @staticmethod
  45. def _ensure_npm():
  46. """Check if the 'npm' command is available in the recipes"""
  47. if not TINFOIL.recipes_parsed:
  48. TINFOIL.parse_recipes()
  49. try:
  50. d = TINFOIL.parse_recipe("nodejs-native")
  51. except bb.providers.NoProvider:
  52. bb.error("Nothing provides 'nodejs-native' which is required for the build")
  53. bb.note("You will likely need to add a layer that provides nodejs")
  54. sys.exit(14)
  55. bindir = d.getVar("STAGING_BINDIR_NATIVE")
  56. npmpath = os.path.join(bindir, "npm")
  57. if not os.path.exists(npmpath):
  58. TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
  59. if not os.path.exists(npmpath):
  60. bb.error("Failed to add 'npm' to sysroot")
  61. sys.exit(14)
  62. return bindir
  63. @staticmethod
  64. def _npm_global_configs(dev):
  65. """Get the npm global configuration"""
  66. configs = []
  67. if dev:
  68. configs.append(("also", "development"))
  69. else:
  70. configs.append(("only", "production"))
  71. configs.append(("save", "false"))
  72. configs.append(("package-lock", "false"))
  73. configs.append(("shrinkwrap", "false"))
  74. return configs
  75. def _run_npm_install(self, d, srctree, registry, dev):
  76. """Run the 'npm install' command without building the addons"""
  77. configs = self._npm_global_configs(dev)
  78. configs.append(("ignore-scripts", "true"))
  79. if registry:
  80. configs.append(("registry", registry))
  81. bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
  82. env = NpmEnvironment(d, configs=configs)
  83. env.run("npm install", workdir=srctree)
  84. def _generate_shrinkwrap(self, d, srctree, dev):
  85. """Check and generate the 'npm-shrinkwrap.json' file if needed"""
  86. configs = self._npm_global_configs(dev)
  87. env = NpmEnvironment(d, configs=configs)
  88. env.run("npm shrinkwrap", workdir=srctree)
  89. return os.path.join(srctree, "npm-shrinkwrap.json")
  90. def _handle_licenses(self, srctree, shrinkwrap_file, dev):
  91. """Return the extra license files and the list of packages"""
  92. licfiles = []
  93. packages = {}
  94. def _licfiles_append(licfile):
  95. """Append 'licfile' to the license files list"""
  96. licfilepath = os.path.join(srctree, licfile)
  97. licmd5 = bb.utils.md5_file(licfilepath)
  98. licfiles.append("file://%s;md5=%s" % (licfile, licmd5))
  99. # Handle the parent package
  100. _licfiles_append("package.json")
  101. packages["${PN}"] = ""
  102. # Handle the dependencies
  103. def _handle_dependency(name, params, deptree):
  104. suffix = "-".join([self._npm_name(dep) for dep in deptree])
  105. destdirs = [os.path.join("node_modules", dep) for dep in deptree]
  106. destdir = os.path.join(*destdirs)
  107. _licfiles_append(os.path.join(destdir, "package.json"))
  108. packages["${PN}-" + suffix] = destdir
  109. with open(shrinkwrap_file, "r") as f:
  110. shrinkwrap = json.load(f)
  111. foreach_dependencies(shrinkwrap, _handle_dependency, dev)
  112. return licfiles, packages
  113. def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
  114. """Handle the npm recipe creation"""
  115. if "buildsystem" in handled:
  116. return False
  117. files = RecipeHandler.checkfiles(srctree, ["package.json"])
  118. if not files:
  119. return False
  120. with open(files[0], "r") as f:
  121. data = json.load(f)
  122. if "name" not in data or "version" not in data:
  123. return False
  124. extravalues["PN"] = self._npm_name(data["name"])
  125. extravalues["PV"] = data["version"]
  126. if "description" in data:
  127. extravalues["SUMMARY"] = data["description"]
  128. if "homepage" in data:
  129. extravalues["HOMEPAGE"] = data["homepage"]
  130. dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
  131. registry = self._get_registry(lines_before)
  132. bb.note("Checking if npm is available ...")
  133. # The native npm is used here (and not the host one) to ensure that the
  134. # npm version is high enough to ensure an efficient dependency tree
  135. # resolution and avoid issue with the shrinkwrap file format.
  136. # Moreover the native npm is mandatory for the build.
  137. bindir = self._ensure_npm()
  138. d = bb.data.createCopy(TINFOIL.config_data)
  139. d.prependVar("PATH", bindir + ":")
  140. d.setVar("S", srctree)
  141. bb.note("Generating shrinkwrap file ...")
  142. # To generate the shrinkwrap file the dependencies have to be installed
  143. # first. During the generation process some files may be updated /
  144. # deleted. By default devtool tracks the diffs in the srctree and raises
  145. # errors when finishing the recipe if some diffs are found.
  146. git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
  147. if os.path.exists(git_exclude_file):
  148. with open(git_exclude_file, "r+") as f:
  149. lines = f.readlines()
  150. for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
  151. if line not in lines:
  152. f.write(line + "\n")
  153. lock_file = os.path.join(srctree, "package-lock.json")
  154. lock_copy = lock_file + ".copy"
  155. if os.path.exists(lock_file):
  156. bb.utils.copyfile(lock_file, lock_copy)
  157. self._run_npm_install(d, srctree, registry, dev)
  158. shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)
  159. if os.path.exists(lock_copy):
  160. bb.utils.movefile(lock_copy, lock_file)
  161. # Add the shrinkwrap file as 'extrafiles'
  162. shrinkwrap_copy = shrinkwrap_file + ".copy"
  163. bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
  164. extravalues.setdefault("extrafiles", {})
  165. extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
  166. url_local = "npmsw://%s" % shrinkwrap_file
  167. url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
  168. if dev:
  169. url_local += ";dev=1"
  170. url_recipe += ";dev=1"
  171. # Add the npmsw url in the SRC_URI of the generated recipe
  172. def _handle_srcuri(varname, origvalue, op, newlines):
  173. """Update the version value and add the 'npmsw://' url"""
  174. value = origvalue.replace("version=" + data["version"], "version=${PV}")
  175. value = value.replace("version=latest", "version=${PV}")
  176. values = [line.strip() for line in value.strip('\n').splitlines()]
  177. values.append(url_recipe)
  178. return values, None, 4, False
  179. (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
  180. lines_before[:] = [line.rstrip('\n') for line in newlines]
  181. # In order to generate correct licence checksums in the recipe the
  182. # dependencies have to be fetched again using the npmsw url
  183. bb.note("Fetching npm dependencies ...")
  184. bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
  185. fetcher = bb.fetch2.Fetch([url_local], d)
  186. fetcher.download()
  187. fetcher.unpack(srctree)
  188. bb.note("Handling licences ...")
  189. (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
  190. extravalues["LIC_FILES_CHKSUM"] = licfiles
  191. split_pkg_licenses(guess_license(srctree, d), packages, lines_after, [])
  192. classes.append("npm")
  193. handled.append("buildsystem")
  194. return True
  195. def register_recipe_handlers(handlers):
  196. """Register the npm handler"""
  197. handlers.append((NpmRecipeHandler(), 60))