123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414 |
- # @file EccCheck.py
- #
- # Copyright (c) 2021, Arm Limited. All rights reserved.<BR>
- # Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
- # SPDX-License-Identifier: BSD-2-Clause-Patent
- ##
- import os
- import shutil
- import re
- import csv
- import xml.dom.minidom
- from typing import List, Dict, Tuple
- import logging
- from io import StringIO
- from edk2toolext.environment import shell_environment
- from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
- from edk2toolext.environment.var_dict import VarDict
- from edk2toollib.utility_functions import RunCmd
- class EccCheck(ICiBuildPlugin):
- """
- A CiBuildPlugin that finds the Ecc issues of newly added code in pull request.
- Configuration options:
- "EccCheck": {
- "ExceptionList": [],
- "IgnoreFiles": []
- },
- """
- FindModifyFile = re.compile(r'\+\+\+ b\/(.*)')
- LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*')
- LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*')
- def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
- """ Provide the testcase name and classname for use in reporting
- testclassname: a descriptive string for the testcase can include whitespace
- classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
- Args:
- packagename: string containing name of package to build
- environment: The VarDict for the test to run in
- Returns:
- a tuple containing the testcase name and the classname
- (testcasename, classname)
- """
- return ("Check for efi coding style for " + packagename, packagename + ".EccCheck")
- ##
- # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin
- #
- # - package is the edk2 path to package. This means workspace/packagepath relative.
- # - edk2path object configured with workspace and packages path
- # - PkgConfig Object (dict) for the pkg
- # - EnvConfig Object
- # - Plugin Manager Instance
- # - Plugin Helper Obj Instance
- # - Junit Logger
- # - output_stream the StringIO output stream from this plugin via logging
- def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
- workspace_path = Edk2pathObj.WorkspacePath
- basetools_path = environment.GetValue("EDK_TOOLS_PATH")
- python_path = os.path.join(basetools_path, "Source", "Python")
- env = shell_environment.GetEnvironment()
- env.set_shell_var('PYTHONPATH', python_path)
- env.set_shell_var('WORKSPACE', workspace_path)
- env.set_shell_var('PACKAGES_PATH', os.pathsep.join(Edk2pathObj.PackagePathList))
- self.ECC_PASS = True
- abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(packagename)
- if abs_pkg_path is None:
- tc.SetSkipped()
- tc.LogStdError("No Package folder {0}".format(abs_pkg_path))
- return 0
- # Create temp directory
- temp_path = os.path.join(workspace_path, 'Build', '.pytool', 'Plugin', 'EccCheck')
- try:
- # Delete temp directory
- if os.path.exists(temp_path):
- shutil.rmtree(temp_path)
- # Copy package being scanned to temp_path
- shutil.copytree (
- abs_pkg_path,
- os.path.join(temp_path, packagename),
- symlinks=True
- )
- # Copy exception.xml to temp_path
- shutil.copyfile (
- os.path.join(basetools_path, "Source", "Python", "Ecc", "exception.xml"),
- os.path.join(temp_path, "exception.xml")
- )
- # Output file to use for git diff operations
- temp_diff_output = os.path.join (temp_path, 'diff.txt')
- self.ApplyConfig(pkgconfig, temp_path, packagename)
- modify_dir_list = self.GetModifyDir(packagename, temp_diff_output)
- patch = self.GetDiff(packagename, temp_diff_output)
- ecc_diff_range = self.GetDiffRange(patch, packagename, temp_path)
- #
- # Use temp_path as working directory when running ECC tool
- #
- self.GenerateEccReport(modify_dir_list, ecc_diff_range, temp_path, basetools_path)
- ecc_log = os.path.join(temp_path, "Ecc.log")
- if self.ECC_PASS:
- # Delete temp directory
- if os.path.exists(temp_path):
- shutil.rmtree(temp_path)
- tc.SetSuccess()
- return 0
- else:
- with open(ecc_log, encoding='utf8') as output:
- ecc_output = output.readlines()
- for line in ecc_output:
- logging.error(line.strip())
- # Delete temp directory
- if os.path.exists(temp_path):
- shutil.rmtree(temp_path)
- tc.SetFailed("EccCheck failed for {0}".format(packagename), "CHECK FAILED")
- return 1
- except KeyboardInterrupt:
- # If EccCheck is interrupted by keybard interrupt, then return failure
- # Delete temp directory
- if os.path.exists(temp_path):
- shutil.rmtree(temp_path)
- tc.SetFailed("EccCheck interrupted for {0}".format(packagename), "CHECK FAILED")
- return 1
- else:
- # If EccCheck fails for any other exception type, raise the exception
- # Delete temp directory
- if os.path.exists(temp_path):
- shutil.rmtree(temp_path)
- tc.SetFailed("EccCheck exception for {0}".format(packagename), "CHECK FAILED")
- raise
- return 1
- def GetDiff(self, pkg: str, temp_diff_output: str) -> List[str]:
- patch = []
- #
- # Generate unified diff between origin/master and HEAD.
- #
- params = "diff --output={} --unified=0 origin/master HEAD".format(temp_diff_output)
- RunCmd("git", params)
- with open(temp_diff_output) as file:
- patch = file.read().strip().split('\n')
- return patch
- def GetModifyDir(self, pkg: str, temp_diff_output: str) -> List[str]:
- #
- # Generate diff between origin/master and HEAD using --diff-filter to
- # exclude deleted and renamed files that do not need to be scanned by
- # ECC. Also use --name-status to only generate the names of the files
- # with differences. The output format of this git diff command is a
- # list of files with the change status and the filename. The filename
- # is always at the end of the line. Examples:
- #
- # M MdeModulePkg/Application/CapsuleApp/CapsuleApp.h
- # M MdeModulePkg/Application/UiApp/FrontPage.h
- #
- params = "diff --output={} --diff-filter=dr --name-status origin/master HEAD".format(temp_diff_output)
- RunCmd("git", params)
- dir_list = []
- with open(temp_diff_output) as file:
- dir_list = file.read().strip().split('\n')
- modify_dir_list = []
- for modify_dir in dir_list:
- #
- # Parse file name from the end of the line
- #
- file_path = modify_dir.strip().split()
- #
- # Skip lines that do not have at least 2 elements (status and file name)
- #
- if len(file_path) < 2:
- continue
- #
- # Parse the directory name from the file name
- #
- file_dir = os.path.dirname(file_path[-1])
- #
- # Skip directory names that do not start with the package being scanned.
- #
- if file_dir.split('/')[0] != pkg:
- continue
- #
- # Skip directory names that are identical to the package being scanned.
- # The assumption here is that there are no source files at the package
- # root. Instead, the only expected files in the package root are
- # EDK II meta data files (DEC, DSC, FDF).
- #
- if file_dir == pkg:
- continue
- #
- # Skip directory names that are already in the modified dir list
- #
- if file_dir in modify_dir_list:
- continue
- #
- # Add the candidate directory to scan to the modified dir list
- #
- modify_dir_list.append(file_dir)
- #
- # Remove duplicates from modify_dir_list
- # Given a folder path, ECC performs a recursive scan of that folder.
- # If a parent and child folder are both present in modify_dir_list,
- # then ECC will perform redudanct scans of source files. In order
- # to prevent redundant scans, if a parent and child folder are both
- # present, then remove all the child folders.
- #
- # For example, if modified_dir_list contains the following elements:
- # MdeModulePkg/Core/Dxe
- # MdeModulePkg/Core/Dxe/Hand
- # MdeModulePkg/Core/Dxe/Mem
- #
- # Then MdeModulePkg/Core/Dxe/Hand and MdeModulePkg/Core/Dxe/Mem should
- # be removed because the files in those folders are covered by a scan
- # of MdeModulePkg/Core/Dxe.
- #
- filtered_list = []
- for dir1 in modify_dir_list:
- Append = True
- for dir2 in modify_dir_list:
- if dir1 == dir2:
- continue
- common = os.path.commonpath([dir1, dir2])
- if os.path.normpath(common) == os.path.normpath(dir2):
- Append = False
- break
- if Append and dir1 not in filtered_list:
- filtered_list.append(dir1)
- return filtered_list
- def GetDiffRange(self, patch_diff: List[str], pkg: str, temp_path: str) -> Dict[str, List[Tuple[int, int]]]:
- IsDelete = True
- StartCheck = False
- range_directory: Dict[str, List[Tuple[int, int]]] = {}
- for line in patch_diff:
- modify_file = self.FindModifyFile.findall(line)
- if modify_file and pkg in modify_file[0] and not StartCheck and os.path.isfile(modify_file[0]):
- modify_file_comment_dic = self.GetCommentRange(modify_file[0], temp_path)
- IsDelete = False
- StartCheck = True
- modify_file_dic = modify_file[0]
- modify_file_dic = modify_file_dic.replace("/", os.sep)
- range_directory[modify_file_dic] = []
- elif line.startswith('--- '):
- StartCheck = False
- elif re.match(self.LineScopePattern, line, re.I) and not IsDelete and StartCheck:
- start_line = self.LineNumRange.search(line).group(1)
- line_range = self.LineNumRange.search(line).group(2)
- if not line_range:
- line_range = '1'
- range_directory[modify_file_dic].append((int(start_line), int(start_line) + int(line_range) - 1))
- for i in modify_file_comment_dic:
- if int(i[0]) <= int(start_line) <= int(i[1]):
- range_directory[modify_file_dic].append(i)
- return range_directory
- def GetCommentRange(self, modify_file: str, temp_path: str) -> List[Tuple[int, int]]:
- comment_range: List[Tuple[int, int]] = []
- modify_file_path = os.path.join(temp_path, modify_file)
- if not os.path.exists (modify_file_path):
- return comment_range
- with open(modify_file_path) as f:
- line_no = 1
- Start = False
- for line in f:
- if line.startswith('/**'):
- start_no = line_no
- Start = True
- if line.startswith('**/') and Start:
- end_no = line_no
- Start = False
- comment_range.append((int(start_no), int(end_no)))
- line_no += 1
- if comment_range and comment_range[0][0] == 1:
- del comment_range[0]
- return comment_range
- def GenerateEccReport(self, modify_dir_list: List[str], ecc_diff_range: Dict[str, List[Tuple[int, int]]],
- temp_path: str, basetools_path: str) -> None:
- ecc_need = False
- ecc_run = True
- config = os.path.normpath(os.path.join(basetools_path, "Source", "Python", "Ecc", "config.ini"))
- exception = os.path.normpath(os.path.join(temp_path, "exception.xml"))
- report = os.path.normpath(os.path.join(temp_path, "Ecc.csv"))
- for modify_dir in modify_dir_list:
- target = os.path.normpath(os.path.join(temp_path, modify_dir))
- logging.info('Run ECC tool for the commit in %s' % modify_dir)
- ecc_need = True
- ecc_params = "-c {0} -e {1} -t {2} -r {3}".format(config, exception, target, report)
- return_code = RunCmd("Ecc", ecc_params, workingdir=temp_path)
- if return_code != 0:
- ecc_run = False
- break
- if not ecc_run:
- logging.error('Fail to run ECC tool')
- self.ParseEccReport(ecc_diff_range, temp_path)
- if not ecc_need:
- logging.info("Doesn't need run ECC check")
- return
- def ParseEccReport(self, ecc_diff_range: Dict[str, List[Tuple[int, int]]], temp_path: str) -> None:
- ecc_log = os.path.join(temp_path, "Ecc.log")
- ecc_csv = os.path.join(temp_path, "Ecc.csv")
- row_lines = []
- ignore_error_code = self.GetIgnoreErrorCode()
- if os.path.exists(ecc_csv):
- with open(ecc_csv) as csv_file:
- reader = csv.reader(csv_file)
- for row in reader:
- for modify_file in ecc_diff_range:
- if modify_file in row[3]:
- for i in ecc_diff_range[modify_file]:
- line_no = int(row[4])
- if i[0] <= line_no <= i[1] and row[1] not in ignore_error_code:
- row[0] = '\nEFI coding style error'
- row[1] = 'Error code: ' + row[1]
- row[3] = 'file: ' + row[3]
- row[4] = 'Line number: ' + row[4]
- row_line = '\n *'.join(row)
- row_lines.append(row_line)
- break
- break
- if row_lines:
- self.ECC_PASS = False
- with open(ecc_log, 'a') as log:
- all_line = '\n'.join(row_lines)
- all_line = all_line + '\n'
- log.writelines(all_line)
- return
- def ApplyConfig(self, pkgconfig: Dict[str, List[str]], temp_path: str, pkg: str) -> None:
- if "IgnoreFiles" in pkgconfig:
- for a in pkgconfig["IgnoreFiles"]:
- a = os.path.join(temp_path, pkg, a)
- a = a.replace(os.sep, "/")
- logging.info("Ignoring Files {0}".format(a))
- if os.path.exists(a):
- if os.path.isfile(a):
- os.remove(a)
- elif os.path.isdir(a):
- shutil.rmtree(a)
- else:
- logging.error("EccCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore files".format(a))
- if "ExceptionList" in pkgconfig:
- exception_list = pkgconfig["ExceptionList"]
- exception_xml = os.path.join(temp_path, "exception.xml")
- try:
- logging.info("Appending exceptions")
- self.AppendException(exception_list, exception_xml)
- except Exception as e:
- logging.error("Fail to apply exceptions")
- raise e
- return
- def AppendException(self, exception_list: List[str], exception_xml: str) -> None:
- error_code_list = exception_list[::2]
- keyword_list = exception_list[1::2]
- dom_tree = xml.dom.minidom.parse(exception_xml)
- root_node = dom_tree.documentElement
- for error_code, keyword in zip(error_code_list, keyword_list):
- customer_node = dom_tree.createElement("Exception")
- keyword_node = dom_tree.createElement("KeyWord")
- keyword_node_text_value = dom_tree.createTextNode(keyword)
- keyword_node.appendChild(keyword_node_text_value)
- customer_node.appendChild(keyword_node)
- error_code_node = dom_tree.createElement("ErrorID")
- error_code_text_value = dom_tree.createTextNode(error_code)
- error_code_node.appendChild(error_code_text_value)
- customer_node.appendChild(error_code_node)
- root_node.appendChild(customer_node)
- with open(exception_xml, 'w') as f:
- dom_tree.writexml(f, indent='', addindent='', newl='\n', encoding='UTF-8')
- return
- def GetIgnoreErrorCode(self) -> set:
- """
- Below are kinds of error code that are accurate in ecc scanning of edk2 level.
- But EccCheck plugin is partial scanning so they are always false positive issues.
- The mapping relationship of error code and error message is listed BaseTools/Sourc/Python/Ecc/EccToolError.py
- """
- ignore_error_code = {
- "10000",
- "10001",
- "10002",
- "10003",
- "10004",
- "10005",
- "10006",
- "10007",
- "10008",
- "10009",
- "10010",
- "10011",
- "10012",
- "10013",
- "10015",
- "10016",
- "10017",
- "10022",
- }
- return ignore_error_code
|