EccCheck.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414
  1. # @file EccCheck.py
  2. #
  3. # Copyright (c) 2021, Arm Limited. All rights reserved.<BR>
  4. # Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
  5. # SPDX-License-Identifier: BSD-2-Clause-Patent
  6. ##
  7. import os
  8. import shutil
  9. import re
  10. import csv
  11. import xml.dom.minidom
  12. from typing import List, Dict, Tuple
  13. import logging
  14. from io import StringIO
  15. from edk2toolext.environment import shell_environment
  16. from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
  17. from edk2toolext.environment.var_dict import VarDict
  18. from edk2toollib.utility_functions import RunCmd
  19. class EccCheck(ICiBuildPlugin):
  20. """
  21. A CiBuildPlugin that finds the Ecc issues of newly added code in pull request.
  22. Configuration options:
  23. "EccCheck": {
  24. "ExceptionList": [],
  25. "IgnoreFiles": []
  26. },
  27. """
  28. FindModifyFile = re.compile(r'\+\+\+ b\/(.*)')
  29. LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*')
  30. LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*')
  31. def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
  32. """ Provide the testcase name and classname for use in reporting
  33. testclassname: a descriptive string for the testcase can include whitespace
  34. classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
  35. Args:
  36. packagename: string containing name of package to build
  37. environment: The VarDict for the test to run in
  38. Returns:
  39. a tuple containing the testcase name and the classname
  40. (testcasename, classname)
  41. """
  42. return ("Check for efi coding style for " + packagename, packagename + ".EccCheck")
  43. ##
  44. # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin
  45. #
  46. # - package is the edk2 path to package. This means workspace/packagepath relative.
  47. # - edk2path object configured with workspace and packages path
  48. # - PkgConfig Object (dict) for the pkg
  49. # - EnvConfig Object
  50. # - Plugin Manager Instance
  51. # - Plugin Helper Obj Instance
  52. # - Junit Logger
  53. # - output_stream the StringIO output stream from this plugin via logging
  54. def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
  55. workspace_path = Edk2pathObj.WorkspacePath
  56. basetools_path = environment.GetValue("EDK_TOOLS_PATH")
  57. python_path = os.path.join(basetools_path, "Source", "Python")
  58. env = shell_environment.GetEnvironment()
  59. env.set_shell_var('PYTHONPATH', python_path)
  60. env.set_shell_var('WORKSPACE', workspace_path)
  61. env.set_shell_var('PACKAGES_PATH', os.pathsep.join(Edk2pathObj.PackagePathList))
  62. self.ECC_PASS = True
  63. abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(packagename)
  64. if abs_pkg_path is None:
  65. tc.SetSkipped()
  66. tc.LogStdError("No Package folder {0}".format(abs_pkg_path))
  67. return 0
  68. # Create temp directory
  69. temp_path = os.path.join(workspace_path, 'Build', '.pytool', 'Plugin', 'EccCheck')
  70. try:
  71. # Delete temp directory
  72. if os.path.exists(temp_path):
  73. shutil.rmtree(temp_path)
  74. # Copy package being scanned to temp_path
  75. shutil.copytree (
  76. abs_pkg_path,
  77. os.path.join(temp_path, packagename),
  78. symlinks=True
  79. )
  80. # Copy exception.xml to temp_path
  81. shutil.copyfile (
  82. os.path.join(basetools_path, "Source", "Python", "Ecc", "exception.xml"),
  83. os.path.join(temp_path, "exception.xml")
  84. )
  85. # Output file to use for git diff operations
  86. temp_diff_output = os.path.join (temp_path, 'diff.txt')
  87. self.ApplyConfig(pkgconfig, temp_path, packagename)
  88. modify_dir_list = self.GetModifyDir(packagename, temp_diff_output)
  89. patch = self.GetDiff(packagename, temp_diff_output)
  90. ecc_diff_range = self.GetDiffRange(patch, packagename, temp_path)
  91. #
  92. # Use temp_path as working directory when running ECC tool
  93. #
  94. self.GenerateEccReport(modify_dir_list, ecc_diff_range, temp_path, basetools_path)
  95. ecc_log = os.path.join(temp_path, "Ecc.log")
  96. if self.ECC_PASS:
  97. # Delete temp directory
  98. if os.path.exists(temp_path):
  99. shutil.rmtree(temp_path)
  100. tc.SetSuccess()
  101. return 0
  102. else:
  103. with open(ecc_log, encoding='utf8') as output:
  104. ecc_output = output.readlines()
  105. for line in ecc_output:
  106. logging.error(line.strip())
  107. # Delete temp directory
  108. if os.path.exists(temp_path):
  109. shutil.rmtree(temp_path)
  110. tc.SetFailed("EccCheck failed for {0}".format(packagename), "CHECK FAILED")
  111. return 1
  112. except KeyboardInterrupt:
  113. # If EccCheck is interrupted by keybard interrupt, then return failure
  114. # Delete temp directory
  115. if os.path.exists(temp_path):
  116. shutil.rmtree(temp_path)
  117. tc.SetFailed("EccCheck interrupted for {0}".format(packagename), "CHECK FAILED")
  118. return 1
  119. else:
  120. # If EccCheck fails for any other exception type, raise the exception
  121. # Delete temp directory
  122. if os.path.exists(temp_path):
  123. shutil.rmtree(temp_path)
  124. tc.SetFailed("EccCheck exception for {0}".format(packagename), "CHECK FAILED")
  125. raise
  126. return 1
  127. def GetDiff(self, pkg: str, temp_diff_output: str) -> List[str]:
  128. patch = []
  129. #
  130. # Generate unified diff between origin/master and HEAD.
  131. #
  132. params = "diff --output={} --unified=0 origin/master HEAD".format(temp_diff_output)
  133. RunCmd("git", params)
  134. with open(temp_diff_output) as file:
  135. patch = file.read().strip().split('\n')
  136. return patch
  137. def GetModifyDir(self, pkg: str, temp_diff_output: str) -> List[str]:
  138. #
  139. # Generate diff between origin/master and HEAD using --diff-filter to
  140. # exclude deleted and renamed files that do not need to be scanned by
  141. # ECC. Also use --name-status to only generate the names of the files
  142. # with differences. The output format of this git diff command is a
  143. # list of files with the change status and the filename. The filename
  144. # is always at the end of the line. Examples:
  145. #
  146. # M MdeModulePkg/Application/CapsuleApp/CapsuleApp.h
  147. # M MdeModulePkg/Application/UiApp/FrontPage.h
  148. #
  149. params = "diff --output={} --diff-filter=dr --name-status origin/master HEAD".format(temp_diff_output)
  150. RunCmd("git", params)
  151. dir_list = []
  152. with open(temp_diff_output) as file:
  153. dir_list = file.read().strip().split('\n')
  154. modify_dir_list = []
  155. for modify_dir in dir_list:
  156. #
  157. # Parse file name from the end of the line
  158. #
  159. file_path = modify_dir.strip().split()
  160. #
  161. # Skip lines that do not have at least 2 elements (status and file name)
  162. #
  163. if len(file_path) < 2:
  164. continue
  165. #
  166. # Parse the directory name from the file name
  167. #
  168. file_dir = os.path.dirname(file_path[-1])
  169. #
  170. # Skip directory names that do not start with the package being scanned.
  171. #
  172. if file_dir.split('/')[0] != pkg:
  173. continue
  174. #
  175. # Skip directory names that are identical to the package being scanned.
  176. # The assumption here is that there are no source files at the package
  177. # root. Instead, the only expected files in the package root are
  178. # EDK II meta data files (DEC, DSC, FDF).
  179. #
  180. if file_dir == pkg:
  181. continue
  182. #
  183. # Skip directory names that are already in the modified dir list
  184. #
  185. if file_dir in modify_dir_list:
  186. continue
  187. #
  188. # Add the candidate directory to scan to the modified dir list
  189. #
  190. modify_dir_list.append(file_dir)
  191. #
  192. # Remove duplicates from modify_dir_list
  193. # Given a folder path, ECC performs a recursive scan of that folder.
  194. # If a parent and child folder are both present in modify_dir_list,
  195. # then ECC will perform redudanct scans of source files. In order
  196. # to prevent redundant scans, if a parent and child folder are both
  197. # present, then remove all the child folders.
  198. #
  199. # For example, if modified_dir_list contains the following elements:
  200. # MdeModulePkg/Core/Dxe
  201. # MdeModulePkg/Core/Dxe/Hand
  202. # MdeModulePkg/Core/Dxe/Mem
  203. #
  204. # Then MdeModulePkg/Core/Dxe/Hand and MdeModulePkg/Core/Dxe/Mem should
  205. # be removed because the files in those folders are covered by a scan
  206. # of MdeModulePkg/Core/Dxe.
  207. #
  208. filtered_list = []
  209. for dir1 in modify_dir_list:
  210. Append = True
  211. for dir2 in modify_dir_list:
  212. if dir1 == dir2:
  213. continue
  214. common = os.path.commonpath([dir1, dir2])
  215. if os.path.normpath(common) == os.path.normpath(dir2):
  216. Append = False
  217. break
  218. if Append and dir1 not in filtered_list:
  219. filtered_list.append(dir1)
  220. return filtered_list
  221. def GetDiffRange(self, patch_diff: List[str], pkg: str, temp_path: str) -> Dict[str, List[Tuple[int, int]]]:
  222. IsDelete = True
  223. StartCheck = False
  224. range_directory: Dict[str, List[Tuple[int, int]]] = {}
  225. for line in patch_diff:
  226. modify_file = self.FindModifyFile.findall(line)
  227. if modify_file and pkg in modify_file[0] and not StartCheck and os.path.isfile(modify_file[0]):
  228. modify_file_comment_dic = self.GetCommentRange(modify_file[0], temp_path)
  229. IsDelete = False
  230. StartCheck = True
  231. modify_file_dic = modify_file[0]
  232. modify_file_dic = modify_file_dic.replace("/", os.sep)
  233. range_directory[modify_file_dic] = []
  234. elif line.startswith('--- '):
  235. StartCheck = False
  236. elif re.match(self.LineScopePattern, line, re.I) and not IsDelete and StartCheck:
  237. start_line = self.LineNumRange.search(line).group(1)
  238. line_range = self.LineNumRange.search(line).group(2)
  239. if not line_range:
  240. line_range = '1'
  241. range_directory[modify_file_dic].append((int(start_line), int(start_line) + int(line_range) - 1))
  242. for i in modify_file_comment_dic:
  243. if int(i[0]) <= int(start_line) <= int(i[1]):
  244. range_directory[modify_file_dic].append(i)
  245. return range_directory
  246. def GetCommentRange(self, modify_file: str, temp_path: str) -> List[Tuple[int, int]]:
  247. comment_range: List[Tuple[int, int]] = []
  248. modify_file_path = os.path.join(temp_path, modify_file)
  249. if not os.path.exists (modify_file_path):
  250. return comment_range
  251. with open(modify_file_path) as f:
  252. line_no = 1
  253. Start = False
  254. for line in f:
  255. if line.startswith('/**'):
  256. start_no = line_no
  257. Start = True
  258. if line.startswith('**/') and Start:
  259. end_no = line_no
  260. Start = False
  261. comment_range.append((int(start_no), int(end_no)))
  262. line_no += 1
  263. if comment_range and comment_range[0][0] == 1:
  264. del comment_range[0]
  265. return comment_range
  266. def GenerateEccReport(self, modify_dir_list: List[str], ecc_diff_range: Dict[str, List[Tuple[int, int]]],
  267. temp_path: str, basetools_path: str) -> None:
  268. ecc_need = False
  269. ecc_run = True
  270. config = os.path.normpath(os.path.join(basetools_path, "Source", "Python", "Ecc", "config.ini"))
  271. exception = os.path.normpath(os.path.join(temp_path, "exception.xml"))
  272. report = os.path.normpath(os.path.join(temp_path, "Ecc.csv"))
  273. for modify_dir in modify_dir_list:
  274. target = os.path.normpath(os.path.join(temp_path, modify_dir))
  275. logging.info('Run ECC tool for the commit in %s' % modify_dir)
  276. ecc_need = True
  277. ecc_params = "-c {0} -e {1} -t {2} -r {3}".format(config, exception, target, report)
  278. return_code = RunCmd("Ecc", ecc_params, workingdir=temp_path)
  279. if return_code != 0:
  280. ecc_run = False
  281. break
  282. if not ecc_run:
  283. logging.error('Fail to run ECC tool')
  284. self.ParseEccReport(ecc_diff_range, temp_path)
  285. if not ecc_need:
  286. logging.info("Doesn't need run ECC check")
  287. return
  288. def ParseEccReport(self, ecc_diff_range: Dict[str, List[Tuple[int, int]]], temp_path: str) -> None:
  289. ecc_log = os.path.join(temp_path, "Ecc.log")
  290. ecc_csv = os.path.join(temp_path, "Ecc.csv")
  291. row_lines = []
  292. ignore_error_code = self.GetIgnoreErrorCode()
  293. if os.path.exists(ecc_csv):
  294. with open(ecc_csv) as csv_file:
  295. reader = csv.reader(csv_file)
  296. for row in reader:
  297. for modify_file in ecc_diff_range:
  298. if modify_file in row[3]:
  299. for i in ecc_diff_range[modify_file]:
  300. line_no = int(row[4])
  301. if i[0] <= line_no <= i[1] and row[1] not in ignore_error_code:
  302. row[0] = '\nEFI coding style error'
  303. row[1] = 'Error code: ' + row[1]
  304. row[3] = 'file: ' + row[3]
  305. row[4] = 'Line number: ' + row[4]
  306. row_line = '\n *'.join(row)
  307. row_lines.append(row_line)
  308. break
  309. break
  310. if row_lines:
  311. self.ECC_PASS = False
  312. with open(ecc_log, 'a') as log:
  313. all_line = '\n'.join(row_lines)
  314. all_line = all_line + '\n'
  315. log.writelines(all_line)
  316. return
  317. def ApplyConfig(self, pkgconfig: Dict[str, List[str]], temp_path: str, pkg: str) -> None:
  318. if "IgnoreFiles" in pkgconfig:
  319. for a in pkgconfig["IgnoreFiles"]:
  320. a = os.path.join(temp_path, pkg, a)
  321. a = a.replace(os.sep, "/")
  322. logging.info("Ignoring Files {0}".format(a))
  323. if os.path.exists(a):
  324. if os.path.isfile(a):
  325. os.remove(a)
  326. elif os.path.isdir(a):
  327. shutil.rmtree(a)
  328. else:
  329. logging.error("EccCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore files".format(a))
  330. if "ExceptionList" in pkgconfig:
  331. exception_list = pkgconfig["ExceptionList"]
  332. exception_xml = os.path.join(temp_path, "exception.xml")
  333. try:
  334. logging.info("Appending exceptions")
  335. self.AppendException(exception_list, exception_xml)
  336. except Exception as e:
  337. logging.error("Fail to apply exceptions")
  338. raise e
  339. return
  340. def AppendException(self, exception_list: List[str], exception_xml: str) -> None:
  341. error_code_list = exception_list[::2]
  342. keyword_list = exception_list[1::2]
  343. dom_tree = xml.dom.minidom.parse(exception_xml)
  344. root_node = dom_tree.documentElement
  345. for error_code, keyword in zip(error_code_list, keyword_list):
  346. customer_node = dom_tree.createElement("Exception")
  347. keyword_node = dom_tree.createElement("KeyWord")
  348. keyword_node_text_value = dom_tree.createTextNode(keyword)
  349. keyword_node.appendChild(keyword_node_text_value)
  350. customer_node.appendChild(keyword_node)
  351. error_code_node = dom_tree.createElement("ErrorID")
  352. error_code_text_value = dom_tree.createTextNode(error_code)
  353. error_code_node.appendChild(error_code_text_value)
  354. customer_node.appendChild(error_code_node)
  355. root_node.appendChild(customer_node)
  356. with open(exception_xml, 'w') as f:
  357. dom_tree.writexml(f, indent='', addindent='', newl='\n', encoding='UTF-8')
  358. return
  359. def GetIgnoreErrorCode(self) -> set:
  360. """
  361. Below are kinds of error code that are accurate in ecc scanning of edk2 level.
  362. But EccCheck plugin is partial scanning so they are always false positive issues.
  363. The mapping relationship of error code and error message is listed BaseTools/Sourc/Python/Ecc/EccToolError.py
  364. """
  365. ignore_error_code = {
  366. "10000",
  367. "10001",
  368. "10002",
  369. "10003",
  370. "10004",
  371. "10005",
  372. "10006",
  373. "10007",
  374. "10008",
  375. "10009",
  376. "10010",
  377. "10011",
  378. "10012",
  379. "10013",
  380. "10015",
  381. "10016",
  382. "10017",
  383. "10022",
  384. }
  385. return ignore_error_code